Support for reading in an apiview_properties.json file in the sources.jar file, for things like typespec back-referencing. (#7078)

This commit is contained in:
Jonathan Giles 2024-03-16 07:20:37 +10:00 коммит произвёл GitHub
Родитель fbff3bf69d
Коммит 2d89dc61b1
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
7 изменённых файлов: 156 добавлений и 239 удалений

Просмотреть файл

@ -3,11 +3,7 @@ package com.azure.tools.apiview.processor;
import com.azure.tools.apiview.processor.analysers.JavaASTAnalyser;
import com.azure.tools.apiview.processor.analysers.Analyser;
import com.azure.tools.apiview.processor.analysers.XMLASTAnalyser;
import com.azure.tools.apiview.processor.model.APIListing;
import com.azure.tools.apiview.processor.model.Diagnostic;
import com.azure.tools.apiview.processor.model.DiagnosticKind;
import com.azure.tools.apiview.processor.model.LanguageVariant;
import com.azure.tools.apiview.processor.model.Token;
import com.azure.tools.apiview.processor.model.*;
import com.azure.tools.apiview.processor.model.maven.Pom;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -16,6 +12,7 @@ import com.fasterxml.jackson.databind.ObjectWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
@ -167,11 +164,26 @@ public class Main {
}
System.out.println(" Using '" + apiListing.getLanguageVariant() + "' for the language variant");
final Analyser analyser = new JavaASTAnalyser(inputFile, apiListing);
final Analyser analyser = new JavaASTAnalyser(apiListing);
// Read all files within the jar file so that we can create a list of files to analyse
final List<Path> allFiles = new ArrayList<>();
try (FileSystem fs = FileSystems.newFileSystem(inputFile.toPath(), Main.class.getClassLoader())) {
try {
// we eagerly load the apiview_properties.json file into an ApiViewProperties object, so that it can
// be used throughout the analysis process, as required
URL apiViewPropertiesFile = fs.getPath("/META-INF/apiview_properties.json").toUri().toURL();
final ObjectMapper objectMapper = new ObjectMapper();
ApiViewProperties properties = objectMapper.readValue(apiViewPropertiesFile, ApiViewProperties.class);
apiListing.setApiViewProperties(properties);
System.out.println(" Found apiview_properties.json file in jar file");
System.out.println(" - Found " + properties.getCrossLanguageDefinitionIds().size() + " cross-language definition IDs");
} catch (Exception e) {
// this is fine, we just won't have any APIView properties to read in
System.out.println(" No apiview_properties.json file found in jar file - continuing...");
}
fs.getRootDirectories().forEach(root -> {
try (Stream<Path> paths = Files.walk(root)) {
paths.forEach(allFiles::add);
@ -183,6 +195,8 @@ public class Main {
// Do the analysis while the filesystem is still represented in memory
analyser.analyse(allFiles);
} catch (Exception e) {
e.printStackTrace();
}
}

Просмотреть файл

@ -39,6 +39,7 @@ import com.github.javaparser.ast.expr.Name;
import com.github.javaparser.ast.expr.NormalAnnotationExpr;
import com.github.javaparser.ast.modules.ModuleDeclaration;
import com.github.javaparser.ast.nodeTypes.NodeWithAnnotations;
import com.github.javaparser.ast.nodeTypes.NodeWithSimpleName;
import com.github.javaparser.ast.nodeTypes.NodeWithType;
import com.github.javaparser.ast.type.ClassOrInterfaceType;
import com.github.javaparser.ast.type.ReferenceType;
@ -73,13 +74,7 @@ import java.util.stream.Stream;
import org.apache.commons.lang.StringEscapeUtils;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.attemptToFindJavadocComment;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.getPackageName;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isInterfaceType;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isPrivateOrPackagePrivate;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isPublicOrProtected;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isTypeAPublicAPI;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.makeId;
import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.*;
import static com.azure.tools.apiview.processor.analysers.util.TokenModifier.INDENT;
import static com.azure.tools.apiview.processor.analysers.util.TokenModifier.NEWLINE;
import static com.azure.tools.apiview.processor.analysers.util.TokenModifier.NOTHING;
@ -111,19 +106,18 @@ public class JavaASTAnalyser implements Analyser {
private static final Pattern SPLIT_NEWLINE = Pattern.compile(MiscUtils.LINEBREAK);
// This is the model that we build up as the AST of all files are analysed. The APIListing is then output as
// JSON that can be understood by APIView.
private final APIListing apiListing;
private final Map<String, JavadocComment> packageNameToPackageInfoJavaDoc;
private final Map<String, JavadocComment> packageNameToPackageInfoJavaDoc = new HashMap<>();
private final Diagnostics diagnostic;
private final Diagnostics diagnostic = new Diagnostics();
private int indent;
private int indent = 0;
public JavaASTAnalyser(File inputFile, APIListing apiListing) {
public JavaASTAnalyser(APIListing apiListing) {
this.apiListing = apiListing;
this.indent = 0;
this.packageNameToPackageInfoJavaDoc = new HashMap<>();
this.diagnostic = new Diagnostics();
}
@Override
@ -183,10 +177,6 @@ public class JavaASTAnalyser implements Analyser {
}
}
public CompilationUnit getCompilationUnit() {
return compilationUnit;
}
public Path getPath() {
return path;
}
@ -209,7 +199,6 @@ public class JavaASTAnalyser implements Analyser {
// Set up a minimal type solver that only looks at the classes used to run this sample.
CombinedTypeSolver combinedTypeSolver = new CombinedTypeSolver();
combinedTypeSolver.add(new ReflectionTypeSolver(false));
// combinedTypeSolver.add(new SourceJarTypeSolver(inputFile));
ParserConfiguration parserConfiguration = new ParserConfiguration()
.setStoreTokens(true)
@ -343,7 +332,7 @@ public class JavaASTAnalyser implements Analyser {
// we don't care to present test scope dependencies
return;
}
String scope = k.equals("") ? "compile" : k;
String scope = k.isEmpty() ? "compile" : k;
addToken(makeWhitespace());
addToken(new Token(COMMENT, "// " + scope + " scope"), NEWLINE);
@ -510,6 +499,23 @@ public class JavaASTAnalyser implements Analyser {
addToken(new Token(TYPE_NAME, moduleDeclaration.getNameAsString(), MODULE_INFO_KEY), SPACE);
addToken(new Token(PUNCTUATION, "{"), NEWLINE);
// Sometimes an exports or opens statement is conditional, so we need to handle that case
// in a single location here, to remove duplication.
Consumer<NodeList<Name>> conditionalExportsToOrOpensToConsumer = names -> {
if (!names.isEmpty()) {
addToken(new Token(WHITESPACE, " "));
addToken(new Token(KEYWORD, "to"), SPACE);
for (int i = 0; i < names.size(); i++) {
addToken(new Token(TYPE_NAME, names.get(i).toString()));
if (i < names.size() - 1) {
addToken(new Token(PUNCTUATION, ","), SPACE);
}
}
}
};
moduleDeclaration.getDirectives().forEach(moduleDirective -> {
indent();
addToken(makeWhitespace());
@ -532,43 +538,14 @@ public class JavaASTAnalyser implements Analyser {
moduleDirective.ifModuleExportsStmt(d -> {
addToken(new Token(KEYWORD, "exports"), SPACE);
addToken(new Token(TYPE_NAME, d.getNameAsString(), makeId(MODULE_INFO_KEY + "-exports-" + d.getNameAsString())));
NodeList<Name> names = d.getModuleNames();
if (!names.isEmpty()) {
addToken(new Token(WHITESPACE, " "));
addToken(new Token(KEYWORD, "to"), SPACE);
for (int i = 0; i < names.size(); i++) {
addToken(new Token(TYPE_NAME, names.get(i).toString()));
if (i < names.size() - 1) {
addToken(new Token(PUNCTUATION, ","), SPACE);
}
}
}
conditionalExportsToOrOpensToConsumer.accept(d.getModuleNames());
addToken(new Token(PUNCTUATION, ";"), NEWLINE);
});
moduleDirective.ifModuleOpensStmt(d -> {
addToken(new Token(KEYWORD, "opens"), SPACE);
addToken(new Token(TYPE_NAME, d.getNameAsString(), makeId(MODULE_INFO_KEY + "-opens-" + d.getNameAsString())));
NodeList<Name> names = d.getModuleNames();
if (names.size() > 0) {
addToken(new Token(WHITESPACE, " "));
addToken(new Token(KEYWORD, "to"), SPACE);
for (int i = 0; i < names.size(); i++) {
addToken(new Token(TYPE_NAME, names.get(i).toString()));
if (i < names.size() - 1) {
addToken(new Token(PUNCTUATION, ","), SPACE);
}
}
}
conditionalExportsToOrOpensToConsumer.accept(d.getModuleNames());
addToken(new Token(PUNCTUATION, ";"), NEWLINE);
});
@ -694,7 +671,13 @@ public class JavaASTAnalyser implements Analyser {
addToken(new Token(DEPRECATED_RANGE_START));
}
addToken(new Token(TYPE_NAME, className, classId));
// setting the class name. We need to look up to see if the apiview_properties.json file specified a
// cross language definition id for this type. If it did, we will use that. The apiview_properties.json
// file uses fully-qualified type names and method names, so we need to ensure that it what we are using
// when we look for a match.
Token typeNameToken = new Token(TYPE_NAME, className, classId);
checkForCrossLanguageDefinitionId(typeNameToken, typeDeclaration);
addToken(typeNameToken);
if (isDeprecated) {
addToken(new Token(DEPRECATED_RANGE_END));
@ -755,6 +738,25 @@ public class JavaASTAnalyser implements Analyser {
addToken(SPACE, new Token(PUNCTUATION, "{"), NEWLINE);
}
/*
* This method is used to add 'cross language definition id' to the token if it is defined in the
* apiview_properties.json file. This is used most commonly in conjunction with TypeSpec-generated libraries,
* so that we may review cross languages with some level of confidence that the types and methods are the same.
*/
private void checkForCrossLanguageDefinitionId(Token typeNameToken, NodeWithSimpleName<?> node) {
Optional<String> fqn;
if (node instanceof TypeDeclaration) {
fqn = ((TypeDeclaration<?>) node).getFullyQualifiedName();
} else if (node instanceof CallableDeclaration) {
fqn = Optional.of(getNodeFullyQualifiedName((CallableDeclaration<?>) node));
} else {
fqn = Optional.empty();
}
fqn.flatMap(_fqn -> apiListing.getApiViewProperties().getCrossLanguageDefinitionId(_fqn))
.ifPresent(typeNameToken::setCrossLanguageDefinitionId);
}
private void tokeniseAnnotationMember(AnnotationDeclaration annotationDeclaration) {
indent();
// Member methods in the annotation declaration
@ -1125,7 +1127,9 @@ public class JavaASTAnalyser implements Analyser {
addToken(new Token(DEPRECATED_RANGE_START));
}
addToken(new Token(MEMBER_NAME, name, definitionId));
Token nameToken = new Token(MEMBER_NAME, name, definitionId);
checkForCrossLanguageDefinitionId(nameToken, callableDeclaration);
addToken(nameToken);
if (isDeprecated) {
addToken(new Token(DEPRECATED_RANGE_END));
@ -1188,7 +1192,7 @@ public class JavaASTAnalyser implements Analyser {
private void getThrowException(CallableDeclaration<?> callableDeclaration) {
final NodeList<ReferenceType> thrownExceptions = callableDeclaration.getThrownExceptions();
if (thrownExceptions.size() == 0) {
if (thrownExceptions.isEmpty()) {
return;
}
@ -1425,7 +1429,7 @@ public class JavaASTAnalyser implements Analyser {
// convert http/s links to external clickable links
Matcher urlMatch = MiscUtils.URL_MATCH.matcher(line2);
int currentIndex = 0;
while(urlMatch.find(currentIndex) == true) {
while(urlMatch.find(currentIndex)) {
int start = urlMatch.start();
int end = urlMatch.end();

Просмотреть файл

@ -333,28 +333,37 @@ public final class ASTUtils {
return false;
}
public static boolean isTypeImplementingInterface(TypeDeclaration type, String interfaceName) {
public static boolean isTypeImplementingInterface(TypeDeclaration<?> type, String interfaceName) {
return type.asClassOrInterfaceDeclaration().getImplementedTypes().stream()
.anyMatch(_interface -> _interface.getNameAsString().equals(interfaceName));
}
private static String getNodeFullyQualifiedName(Optional<Node> nodeOptional) {
if (!nodeOptional.isPresent()) {
public static String getNodeFullyQualifiedName(Node node) {
if (node == null) {
return "";
}
Node node = nodeOptional.get();
if (node instanceof TypeDeclaration<?>) {
TypeDeclaration<?> type = (TypeDeclaration<?>) node;
return type.getFullyQualifiedName().get();
} else if (node instanceof CallableDeclaration) {
CallableDeclaration callableDeclaration = (CallableDeclaration) node;
return getNodeFullyQualifiedName(node.getParentNode()) + "." + callableDeclaration.getNameAsString();
CallableDeclaration<?> callableDeclaration = (CallableDeclaration<?>) node;
String fqn = getNodeFullyQualifiedName(node.getParentNode()) + "." + callableDeclaration.getNameAsString();
if (callableDeclaration.isConstructorDeclaration()) {
fqn += ".ctor";
}
return fqn;
} else {
return "";
}
}
private static String getNodeFullyQualifiedName(Optional<Node> nodeOptional) {
return nodeOptional.map(ASTUtils::getNodeFullyQualifiedName).orElse("");
}
/**
* Attempts to retrieve the {@link JavadocComment} for a given {@link BodyDeclaration}.
* <p>

Просмотреть файл

@ -1,172 +0,0 @@
package com.azure.tools.apiview.processor.analysers.util;
import com.github.javaparser.StaticJavaParser;
import com.github.javaparser.ast.CompilationUnit;
import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration;
import com.github.javaparser.ast.body.TypeDeclaration;
import com.github.javaparser.resolution.UnsolvedSymbolException;
import com.github.javaparser.resolution.declarations.ResolvedReferenceTypeDeclaration;
import com.github.javaparser.symbolsolver.javaparsermodel.declarations.JavaParserClassDeclaration;
import com.github.javaparser.symbolsolver.javaparsermodel.declarations.JavaParserEnumDeclaration;
import com.github.javaparser.symbolsolver.javaparsermodel.declarations.JavaParserInterfaceDeclaration;
import com.github.javaparser.symbolsolver.model.resolution.SymbolReference;
import com.github.javaparser.symbolsolver.model.resolution.TypeSolver;
import javassist.ClassPool;
import javassist.NotFoundException;
import java.io.*;
import java.nio.file.Path;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
/**
* Will let the symbol solver look inside a jar file while solving types.
*/
public class SourceJarTypeSolver implements TypeSolver {
private static SourceJarTypeSolver instance;
private TypeSolver parent;
private Map<String, ClasspathElement> classpathElements = new HashMap<>();
private ClassPool classPool = new ClassPool(false);
public SourceJarTypeSolver(Path pathToJar) throws IOException {
this(pathToJar.toFile());
}
public SourceJarTypeSolver(File pathToJar) throws IOException {
this(pathToJar.getCanonicalPath());
}
public SourceJarTypeSolver(String pathToJar) throws IOException {
addPathToJar(pathToJar);
}
public SourceJarTypeSolver(InputStream jarInputStream) throws IOException {
addPathToJar(jarInputStream);
}
public static SourceJarTypeSolver getJarTypeSolver(String pathToJar) throws IOException {
if (instance == null) {
instance = new SourceJarTypeSolver(pathToJar);
} else {
instance.addPathToJar(pathToJar);
}
return instance;
}
private File dumpToTempFile(InputStream inputStream) throws IOException {
File tempFile = File.createTempFile("jar_file_from_input_stream", ".jar");
tempFile.deleteOnExit();
byte[] buffer = new byte[8 * 1024];
try (OutputStream output = new FileOutputStream(tempFile)) {
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
output.write(buffer, 0, bytesRead);
}
} finally {
inputStream.close();
}
return tempFile;
}
private void addPathToJar(InputStream jarInputStream) throws IOException {
addPathToJar(dumpToTempFile(jarInputStream).getAbsolutePath());
}
private void addPathToJar(String pathToJar) throws IOException {
try {
classPool.appendClassPath(pathToJar);
classPool.appendSystemPath();
} catch (NotFoundException e) {
throw new RuntimeException(e);
}
JarFile jarFile = new JarFile(pathToJar);
JarEntry entry;
Enumeration<JarEntry> e = jarFile.entries();
while (e.hasMoreElements()) {
entry = e.nextElement();
if (entry != null && !entry.isDirectory() && entry.getName().endsWith(".java")) {
String name = entryPathToClassName(entry.getName());
classpathElements.put(name, new ClasspathElement(jarFile, entry));
}
}
}
@Override
public TypeSolver getParent() {
return parent;
}
@Override
public void setParent(TypeSolver parent) {
this.parent = parent;
}
private String entryPathToClassName(String entryPath) {
if (!entryPath.endsWith(".java")) {
throw new IllegalStateException();
}
String className = entryPath.substring(0, entryPath.length() - ".java".length());
className = className.replace('/', '.');
className = className.replace('$', '.');
return className;
}
@Override
public SymbolReference<ResolvedReferenceTypeDeclaration> tryToSolveType(String name) {
if (classpathElements.containsKey(name)) {
CompilationUnit cu = classpathElements.get(name).parseJava();
for (TypeDeclaration<?> type : cu.getTypes()) {
if (type.isClassOrInterfaceDeclaration()) {
ClassOrInterfaceDeclaration classType = (ClassOrInterfaceDeclaration) type;
return SymbolReference.solved(type.asClassOrInterfaceDeclaration().isInterface() ?
new JavaParserInterfaceDeclaration(classType, this) :
new JavaParserClassDeclaration(classType, this));
} else if (type.isEnumDeclaration()) {
return SymbolReference.solved(new JavaParserEnumDeclaration(type.asEnumDeclaration(), this));
} else {
System.err.println("Can't resolve " + type);
}
}
return SymbolReference.unsolved(ResolvedReferenceTypeDeclaration.class);
} else {
return SymbolReference.unsolved(ResolvedReferenceTypeDeclaration.class);
}
}
@Override
public ResolvedReferenceTypeDeclaration solveType(String name) throws UnsolvedSymbolException {
SymbolReference<ResolvedReferenceTypeDeclaration> ref = tryToSolveType(name);
if (ref.isSolved()) {
return ref.getCorrespondingDeclaration();
} else {
throw new UnsolvedSymbolException(name);
}
}
private class ClasspathElement {
private JarFile jarFile;
private JarEntry entry;
ClasspathElement(JarFile jarFile, JarEntry entry) {
this.jarFile = jarFile;
this.entry = entry;
}
public CompilationUnit parseJava() {
try {
return StaticJavaParser.parse(jarFile.getInputStream(entry));
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
}
}

Просмотреть файл

@ -56,12 +56,16 @@ public class APIListing {
@JsonIgnore
private Pom mavenPom;
@JsonIgnore
private ApiViewProperties apiViewProperties;
public APIListing() {
this.diagnostics = new ArrayList<>();
this.knownTypes = new HashMap<>();
this.packageNamesToTypesMap = new HashMap<>();
this.typeToPackageNameMap = new HashMap<>();
this.navigation = new ArrayList<>();
this.apiViewProperties = new ApiViewProperties();
}
public void setReviewName(final String name) {
@ -158,4 +162,12 @@ public class APIListing {
public Pom getMavenPom() {
return mavenPom;
}
public void setApiViewProperties(ApiViewProperties properties) {
this.apiViewProperties = properties;
}
public ApiViewProperties getApiViewProperties() {
return apiViewProperties;
}
}

Просмотреть файл

@ -0,0 +1,35 @@
package com.azure.tools.apiview.processor.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
/**
* Sometimes libraries carry additional metadata with them that can make the output from APIView more useful. This
* class is used to store that metadata, as it is deserialized from the /META-INF/apiview_properties.json file.
*/
public class ApiViewProperties {
// This is a map of model names and methods to their TypeSpec definition IDs.
@JsonProperty("CrossLanguageDefinitionId")
private final Map<String, String> crossLanguageDefinitionIds = new HashMap<>();
/**
* Cross Languages Definition ID is used to map from a model name or a method name to a TypeSpec definition ID. This
* is used to enable cross-language linking, to make review time quicker as reviewers can jump between languages to
* see how the API is implemented in each language.
*/
public Optional<String> getCrossLanguageDefinitionId(String fullyQualifiedName) {
return Optional.ofNullable(crossLanguageDefinitionIds.get(fullyQualifiedName));
}
/**
* Returns an unmodifiable map of all the cross-language definition IDs.
*/
public Map<String, String> getCrossLanguageDefinitionIds() {
return Collections.unmodifiableMap(crossLanguageDefinitionIds);
}
}

Просмотреть файл

@ -15,6 +15,9 @@ public class Token {
@JsonProperty("Value")
private String value;
@JsonProperty("CrossLanguageDefinitionId")
private String crossLanguageDefinitionId;
public Token(final TokenKind kind) {
this(kind, null);
}
@ -37,6 +40,18 @@ public class Token {
this.definitionId = definitionId;
}
public String getCrossLanguageDefinitionId() {
return crossLanguageDefinitionId;
}
/**
* This is used to link tokens back to TypeSpec definitions, and therefore, to other languages that have been
* generated from the same TypeSpec.
*/
public void setCrossLanguageDefinitionId(String crossLanguageDefinitionId) {
this.crossLanguageDefinitionId = crossLanguageDefinitionId;
}
public String getNavigateToId() {
return navigateToId;
}