From 4419b675577d7c281a659fab1fec10e1bfbe04c5 Mon Sep 17 00:00:00 2001 From: Douglas Gregor Date: Thu, 21 Oct 2010 06:10:04 +0000 Subject: [PATCH] Improve the implementation of libclang's token-annotation logic for entities in the preprocessing record. Previously, we would only end up getting the first token of a preprocessing record annotated correctly. For example, given #include "foo.h" we would only get the '#' annotated as an inclusion directive; the 'include' and '"foo.h"' tokens would be given the general 'processing directive' annotation. Now, we get proper annotations for entities in the preprocessing record. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@117001 91177308-0d34-0410-b5e6-96231b3b80d8 --- test/Index/annotate-tokens-include.c | 4 +- test/Index/annotate-tokens-pp.c | 96 ++++++++++++++-------------- test/Index/annotate-tokens.m | 2 +- tools/libclang/CIndex.cpp | 84 +++++++++++++++++++----- 4 files changed, 120 insertions(+), 66 deletions(-) diff --git a/test/Index/annotate-tokens-include.c b/test/Index/annotate-tokens-include.c index b60c1af181..2190125c09 100644 --- a/test/Index/annotate-tokens-include.c +++ b/test/Index/annotate-tokens-include.c @@ -2,6 +2,6 @@ // RUN: c-index-test -test-annotate-tokens=%s:1:1:2:1 %s | FileCheck %s // CHECK: Punctuation: "#" [1:1 - 1:2] inclusion directive=annotate-tokens-include.h -// CHECK: Identifier: "include" [1:2 - 1:9] preprocessing directive= -// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] preprocessing directive= +// CHECK: Identifier: "include" [1:2 - 1:9] inclusion directive=annotate-tokens-include.h +// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] inclusion directive=annotate-tokens-include.h diff --git a/test/Index/annotate-tokens-pp.c b/test/Index/annotate-tokens-pp.c index 631caab839..a6e7fb99d6 100644 --- a/test/Index/annotate-tokens-pp.c +++ b/test/Index/annotate-tokens-pp.c @@ -29,27 +29,27 @@ void test() { // CHECK: Punctuation: "#" [2:1 - 2:2] preprocessing directive= // CHECK: Identifier: "define" [2:2 - 2:8] preprocessing directive= // CHECK: Identifier: "STILL_NOTHING" [2:9 - 2:22] macro definition=STILL_NOTHING -// CHECK: Identifier: "NOTHING" [2:23 - 2:30] preprocessing directive= -// CHECK: Punctuation: "(" [2:30 - 2:31] preprocessing directive= -// CHECK: Identifier: "honk" [2:31 - 2:35] preprocessing directive= -// CHECK: Punctuation: "," [2:35 - 2:36] preprocessing directive= -// CHECK: Identifier: "warble" [2:36 - 2:42] preprocessing directive= -// CHECK: Punctuation: ")" [2:42 - 2:43] preprocessing directive= +// CHECK: Identifier: "NOTHING" [2:23 - 2:30] macro definition=STILL_NOTHING +// CHECK: Punctuation: "(" [2:30 - 2:31] macro definition=STILL_NOTHING +// CHECK: Identifier: "honk" [2:31 - 2:35] macro definition=STILL_NOTHING +// CHECK: Punctuation: "," [2:35 - 2:36] macro definition=STILL_NOTHING +// CHECK: Identifier: "warble" [2:36 - 2:42] macro definition=STILL_NOTHING +// CHECK: Punctuation: ")" [2:42 - 2:43] macro definition=STILL_NOTHING // CHECK: Punctuation: "#" [3:1 - 3:2] preprocessing directive= // CHECK: Identifier: "define" [3:2 - 3:8] preprocessing directive= // CHECK: Identifier: "BAR" [3:9 - 3:12] macro definition=BAR -// CHECK: Identifier: "baz" [3:13 - 3:16] preprocessing directive= +// CHECK: Identifier: "baz" [3:13 - 3:16] macro definition=BAR // CHECK: Punctuation: "#" [4:1 - 4:2] preprocessing directive= // CHECK: Identifier: "define" [4:2 - 4:8] preprocessing directive= // CHECK: Identifier: "WIBBLE" [4:9 - 4:15] macro definition=WIBBLE -// CHECK: Punctuation: "(" [4:15 - 4:16] preprocessing directive= -// CHECK: Identifier: "X" [4:16 - 4:17] preprocessing directive= -// CHECK: Punctuation: "," [4:17 - 4:18] preprocessing directive= -// CHECK: Identifier: "Y" [4:19 - 4:20] preprocessing directive= -// CHECK: Punctuation: ")" [4:20 - 4:21] preprocessing directive= -// CHECK: Identifier: "X" [4:22 - 4:23] preprocessing directive= -// CHECK: Punctuation: "##" [4:23 - 4:25] preprocessing directive= -// CHECK: Identifier: "Y" [4:25 - 4:26] preprocessing directive= +// CHECK: Punctuation: "(" [4:15 - 4:16] macro definition=WIBBLE +// CHECK: Identifier: "X" [4:16 - 4:17] macro definition=WIBBLE +// CHECK: Punctuation: "," [4:17 - 4:18] macro definition=WIBBLE +// CHECK: Identifier: "Y" [4:19 - 4:20] macro definition=WIBBLE +// CHECK: Punctuation: ")" [4:20 - 4:21] macro definition=WIBBLE +// CHECK: Identifier: "X" [4:22 - 4:23] macro definition=WIBBLE +// CHECK: Punctuation: "##" [4:23 - 4:25] macro definition=WIBBLE +// CHECK: Identifier: "Y" [4:25 - 4:26] macro definition=WIBBLE // CHECK: Identifier: "NOTHING" [5:1 - 5:8] macro instantiation=NOTHING:1:9 // CHECK: Punctuation: "(" [5:8 - 5:9] // CHECK: Identifier: "more" [5:9 - 5:13] @@ -69,31 +69,31 @@ void test() { // CHECK: Identifier: "STILL_NOTHING" [6:9 - 6:22] macro instantiation=STILL_NOTHING:2:9 // CHECK: Punctuation: ";" [6:22 - 6:23] // CHECK: Punctuation: "#" [7:1 - 7:2] inclusion directive=foo.h -// CHECK: Identifier: "include" [7:2 - 7:9] preprocessing directive= -// CHECK: Literal: ""foo.h"" [7:10 - 7:17] preprocessing directive= +// CHECK: Identifier: "include" [7:2 - 7:9] inclusion directive=foo.h +// CHECK: Literal: ""foo.h"" [7:10 - 7:17] inclusion directive=foo.h // CHECK: Punctuation: "#" [8:1 - 8:2] preprocessing directive= // CHECK: Identifier: "undef" [8:2 - 8:7] preprocessing directive= // CHECK: Identifier: "BAR" [8:8 - 8:11] preprocessing directive= // CHECK: Punctuation: "#" [10:1 - 10:2] preprocessing directive= // CHECK: Identifier: "define" [10:2 - 10:8] preprocessing directive= // CHECK: Identifier: "REVERSE_MACRO" [10:9 - 10:22] macro definition=REVERSE_MACRO -// CHECK: Punctuation: "(" [10:22 - 10:23] preprocessing directive= -// CHECK: Identifier: "x" [10:23 - 10:24] preprocessing directive= -// CHECK: Punctuation: "," [10:24 - 10:25] preprocessing directive= -// CHECK: Identifier: "y" [10:25 - 10:26] preprocessing directive= -// CHECK: Punctuation: ")" [10:26 - 10:27] preprocessing directive= -// CHECK: Identifier: "y" [10:28 - 10:29] preprocessing directive= -// CHECK: Punctuation: "+" [10:30 - 10:31] preprocessing directive= -// CHECK: Identifier: "x" [10:32 - 10:33] preprocessing directive= +// CHECK: Punctuation: "(" [10:22 - 10:23] macro definition=REVERSE_MACRO +// CHECK: Identifier: "x" [10:23 - 10:24] macro definition=REVERSE_MACRO +// CHECK: Punctuation: "," [10:24 - 10:25] macro definition=REVERSE_MACRO +// CHECK: Identifier: "y" [10:25 - 10:26] macro definition=REVERSE_MACRO +// CHECK: Punctuation: ")" [10:26 - 10:27] macro definition=REVERSE_MACRO +// CHECK: Identifier: "y" [10:28 - 10:29] macro definition=REVERSE_MACRO +// CHECK: Punctuation: "+" [10:30 - 10:31] macro definition=REVERSE_MACRO +// CHECK: Identifier: "x" [10:32 - 10:33] macro definition=REVERSE_MACRO // CHECK: Punctuation: "#" [11:1 - 11:2] preprocessing directive= // CHECK: Identifier: "define" [11:2 - 11:8] preprocessing directive= // CHECK: Identifier: "TWICE_MACRO" [11:9 - 11:20] macro definition=TWICE_MACRO -// CHECK: Punctuation: "(" [11:20 - 11:21] preprocessing directive= -// CHECK: Identifier: "y" [11:21 - 11:22] preprocessing directive= -// CHECK: Punctuation: ")" [11:22 - 11:23] preprocessing directive= -// CHECK: Identifier: "y" [11:24 - 11:25] preprocessing directive= -// CHECK: Punctuation: "+" [11:26 - 11:27] preprocessing directive= -// CHECK: Identifier: "y" [11:28 - 11:29] preprocessing directive= +// CHECK: Punctuation: "(" [11:20 - 11:21] macro definition=TWICE_MACRO +// CHECK: Identifier: "y" [11:21 - 11:22] macro definition=TWICE_MACRO +// CHECK: Punctuation: ")" [11:22 - 11:23] macro definition=TWICE_MACRO +// CHECK: Identifier: "y" [11:24 - 11:25] macro definition=TWICE_MACRO +// CHECK: Punctuation: "+" [11:26 - 11:27] macro definition=TWICE_MACRO +// CHECK: Identifier: "y" [11:28 - 11:29] macro definition=TWICE_MACRO // CHECK: Keyword: "void" [13:1 - 13:5] FunctionDecl=test_macro_args:13:6 (Definition) // CHECK: Identifier: "test_macro_args" [13:6 - 13:21] FunctionDecl=test_macro_args:13:6 (Definition) // CHECK: Punctuation: "(" [13:21 - 13:22] FunctionDecl=test_macro_args:13:6 (Definition) @@ -140,23 +140,23 @@ void test() { // CHECK: Punctuation: "#" [21:1 - 21:2] preprocessing directive= // CHECK: Identifier: "define" [21:2 - 21:8] preprocessing directive= // CHECK: Identifier: "fun_with_macro_bodies" [21:9 - 21:30] macro definition=fun_with_macro_bodies -// CHECK: Punctuation: "(" [21:30 - 21:31] preprocessing directive= -// CHECK: Identifier: "x" [21:31 - 21:32] preprocessing directive= -// CHECK: Punctuation: "," [21:32 - 21:33] preprocessing directive= -// CHECK: Identifier: "y" [21:34 - 21:35] preprocessing directive= -// CHECK: Punctuation: ")" [21:35 - 21:36] preprocessing directive= -// CHECK: Keyword: "do" [21:37 - 21:39] preprocessing directive= -// CHECK: Punctuation: "{" [21:40 - 21:41] preprocessing directive= -// CHECK: Keyword: "if" [21:42 - 21:44] preprocessing directive= -// CHECK: Punctuation: "(" [21:45 - 21:46] preprocessing directive= -// CHECK: Identifier: "x" [21:46 - 21:47] preprocessing directive= -// CHECK: Punctuation: ")" [21:47 - 21:48] preprocessing directive= -// CHECK: Identifier: "y" [21:49 - 21:50] preprocessing directive= -// CHECK: Punctuation: "}" [21:51 - 21:52] preprocessing directive= -// CHECK: Keyword: "while" [21:53 - 21:58] preprocessing directive= -// CHECK: Punctuation: "(" [21:59 - 21:60] preprocessing directive= -// CHECK: Literal: "0" [21:60 - 21:61] preprocessing directive= -// CHECK: Punctuation: ")" [21:61 - 21:62] preprocessing directive= +// CHECK: Punctuation: "(" [21:30 - 21:31] macro definition=fun_with_macro_bodies +// CHECK: Identifier: "x" [21:31 - 21:32] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: "," [21:32 - 21:33] macro definition=fun_with_macro_bodies +// CHECK: Identifier: "y" [21:34 - 21:35] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: ")" [21:35 - 21:36] macro definition=fun_with_macro_bodies +// CHECK: Keyword: "do" [21:37 - 21:39] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: "{" [21:40 - 21:41] macro definition=fun_with_macro_bodies +// CHECK: Keyword: "if" [21:42 - 21:44] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: "(" [21:45 - 21:46] macro definition=fun_with_macro_bodies +// CHECK: Identifier: "x" [21:46 - 21:47] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: ")" [21:47 - 21:48] macro definition=fun_with_macro_bodies +// CHECK: Identifier: "y" [21:49 - 21:50] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: "}" [21:51 - 21:52] macro definition=fun_with_macro_bodies +// CHECK: Keyword: "while" [21:53 - 21:58] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: "(" [21:59 - 21:60] macro definition=fun_with_macro_bodies +// CHECK: Literal: "0" [21:60 - 21:61] macro definition=fun_with_macro_bodies +// CHECK: Punctuation: ")" [21:61 - 21:62] macro definition=fun_with_macro_bodies // CHECK: Keyword: "void" [23:1 - 23:5] FunctionDecl=test:23:6 (Definition) // CHECK: Identifier: "test" [23:6 - 23:10] FunctionDecl=test:23:6 (Definition) // CHECK: Punctuation: "(" [23:10 - 23:11] FunctionDecl=test:23:6 (Definition) diff --git a/test/Index/annotate-tokens.m b/test/Index/annotate-tokens.m index ec919fa164..6b8a39c55d 100644 --- a/test/Index/annotate-tokens.m +++ b/test/Index/annotate-tokens.m @@ -285,7 +285,7 @@ void f() { // CHECK: Punctuation: "#" [63:1 - 63:2] preprocessing directive= // CHECK: Identifier: "define" [63:2 - 63:8] preprocessing directive= // CHECK: Identifier: "VAL" [63:9 - 63:12] macro definition=VAL -// CHECK: Literal: "0" [63:13 - 63:14] preprocessing directive= +// CHECK: Literal: "0" [63:13 - 63:14] macro definition=VAL // CHECK: Punctuation: "@" [65:1 - 65:2] ObjCInterfaceDecl=R7974151:65:12 // CHECK: Keyword: "interface" [65:2 - 65:11] ObjCInterfaceDecl=R7974151:65:12 // CHECK: Identifier: "R7974151" [65:12 - 65:20] ObjCInterfaceDecl=R7974151:65:12 diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp index c944883fa4..b9147fa1a1 100644 --- a/tools/libclang/CIndex.cpp +++ b/tools/libclang/CIndex.cpp @@ -3657,6 +3657,7 @@ class AnnotateTokensWorker { CXCursor *Cursors; unsigned NumTokens; unsigned TokIdx; + unsigned PreprocessingTokIdx; CursorVisitor AnnotateVis; SourceManager &SrcMgr; @@ -3672,7 +3673,7 @@ public: CXToken *tokens, CXCursor *cursors, unsigned numTokens, ASTUnit *CXXUnit, SourceRange RegionOfInterest) : Annotated(annotated), Tokens(tokens), Cursors(cursors), - NumTokens(numTokens), TokIdx(0), + NumTokens(numTokens), TokIdx(0), PreprocessingTokIdx(0), AnnotateVis(CXXUnit, AnnotateTokensVisitor, this, Decl::MaxPCHLevel, RegionOfInterest), SrcMgr(CXXUnit->getSourceManager()) {} @@ -3690,7 +3691,9 @@ void AnnotateTokensWorker::AnnotateTokens(CXCursor parent) { for (unsigned I = 0 ; I < TokIdx ; ++I) { AnnotateTokensData::iterator Pos = Annotated.find(Tokens[I].int_data[1]); - if (Pos != Annotated.end()) + if (Pos != Annotated.end() && + (clang_isInvalid(Cursors[I].kind) || + Pos->second.kind != CXCursor_PreprocessingDirective)) Cursors[I] = Pos->second; } @@ -3706,16 +3709,66 @@ void AnnotateTokensWorker::AnnotateTokens(CXCursor parent) { } enum CXChildVisitResult -AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) { +AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) { CXSourceLocation Loc = clang_getCursorLocation(cursor); - // We can always annotate a preprocessing directive/macro instantiation. - if (clang_isPreprocessing(cursor.kind)) { - Annotated[Loc.int_data] = cursor; + SourceRange cursorRange = getRawCursorExtent(cursor); + + if (clang_isPreprocessing(cursor.kind)) { + // For macro instantiations, just note where the beginning of the macro + // instantiation occurs. + if (cursor.kind == CXCursor_MacroInstantiation) { + Annotated[Loc.int_data] = cursor; + return CXChildVisit_Recurse; + } + + if (cursorRange.isInvalid()) + return CXChildVisit_Continue; + + // Items in the preprocessing record are kept separate from items in + // declarations, so we keep a separate token index. + unsigned SavedTokIdx = TokIdx; + TokIdx = PreprocessingTokIdx; + + // Skip tokens up until we catch up to the beginning of the preprocessing + // entry. + while (MoreTokens()) { + const unsigned I = NextToken(); + SourceLocation TokLoc = GetTokenLoc(I); + switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) { + case RangeBefore: + AdvanceToken(); + continue; + case RangeAfter: + case RangeOverlap: + break; + } + break; + } + + // Look at all of the tokens within this range. + while (MoreTokens()) { + const unsigned I = NextToken(); + SourceLocation TokLoc = GetTokenLoc(I); + switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) { + case RangeBefore: + assert(0 && "Infeasible"); + case RangeAfter: + break; + case RangeOverlap: + Cursors[I] = cursor; + AdvanceToken(); + continue; + } + break; + } + + // Save the preprocessing token index; restore the non-preprocessing + // token index. + PreprocessingTokIdx = TokIdx; + TokIdx = SavedTokIdx; return CXChildVisit_Recurse; } - SourceRange cursorRange = getRawCursorExtent(cursor); - if (cursorRange.isInvalid()) return CXChildVisit_Continue; @@ -3753,7 +3806,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) { // directive. Here we assume that the default construction of CXCursor // results in CXCursor.kind being an initialized value (i.e., 0). If // this isn't the case, we can fix by doing lookup + insertion. - + CXCursor &oldC = Annotated[rawEncoding]; if (!clang_isPreprocessing(oldC.kind)) oldC = cursor; @@ -3815,6 +3868,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) { for (unsigned I = BeforeChildren; I != AfterChildren; ++I) { if (!clang_isInvalid(clang_getCursorKind(Cursors[I]))) break; + Cursors[I] = cursor; } // Scan the tokens that are at the end of the cursor, but are not captured @@ -3841,14 +3895,14 @@ void clang_annotateTokens(CXTranslationUnit TU, if (NumTokens == 0 || !Tokens || !Cursors) return; + // Any token we don't specifically annotate will have a NULL cursor. + CXCursor C = clang_getNullCursor(); + for (unsigned I = 0; I != NumTokens; ++I) + Cursors[I] = C; + ASTUnit *CXXUnit = static_cast(TU); - if (!CXXUnit) { - // Any token we don't specifically annotate will have a NULL cursor. - const CXCursor &C = clang_getNullCursor(); - for (unsigned I = 0; I != NumTokens; ++I) - Cursors[I] = C; + if (!CXXUnit) return; - } ASTUnit::ConcurrencyCheck Check(*CXXUnit);