Bug 1352177 - 2. Introduce new config file format for SDK bindings; r=snorp

Instead of specifying a class name per line, the new format uses the
.ini format, with each section name specifying the class, and each
property name specifying a member of the class. WrapForJNI options can
be specified with each class or member. Comments can be specified with
';' or '#'. For example,

 # Generate bindings for Bundle using default options:
 [android.os.Bundle]

 # Generate bindings for Bundle using class options:
 [android.os.Bundle = exceptionMode:nsresult]

 # Generate bindings for Bundle using method options:
 [android.os.Bundle]
 putInt = stubName:PutInteger

 # Generate bindings for Bundle using class options with method override:
 # (note that all options are overriden at the same time.)
 [android.os.Bundle = exceptionMode:nsresult]
 # putInt will have stubName "PutInteger", and exceptionMode of "abort"
 putInt = stubName:PutInteger
 # putChar will have stubName "PutCharacter", and exceptionMode of "nsresult"
 putChar = stubName:PutCharacter, exceptionMode:nsresult

 # Overloded methods can be specified using its signature
 [android.os.Bundle]
 # Skip the copy constructor
 <init>(Landroid/os/Bundle;)V = skip:true

 # Generic member types can be specified
 [android.view.KeyEvent = skip:true]
 # Skip everything except fields
 <field> = skip:false

 # Skip everything except putInt and putChar
 [android.os.Bundle = skip:true]
 putInt = skip:false
 putChar =

 # Avoid conflicts in native bindings
 [android.os.Bundle]
 # Bundle(PersistableBundle) native binding can conflict with Bundle(ClassLoader)
 <init>(Landroid/os/PersistableBundle;)V = stubName:NewFromPersistableBundle

 # Generate a getter instead of a literal for certain runtime constants
 [android.os.Build$VERSION = skip:true]
 SDK_INT = noLiteral:true

build/annotationProcessors/CodeGenerator.java

@@ -417,9 +417,9 @@ public class CodeGenerator {
         final String uniqueName = info.wrapperName;
         final Class<?> type = field.getType();
 
-        // Handles a peculiar case when dealing with enum types. We don't care about this field.
-        // It just gets in the way and stops our code from compiling.
-        if (field.isSynthetic() || field.getName().equals("$VALUES")) {
+        // Handle various cases where we don't care about the field.
+        if (field.isSynthetic() || field.getName().equals("$VALUES") ||
+                field.getName().equals("CREATOR")) {
             return;
         }
 
@@ -537,35 +537,6 @@ public class CodeGenerator {
                 "\n");
     }
 
-    public void generateMembers(Member[] members) {
-        for (Member m : members) {
-            if (!Modifier.isPublic(m.getModifiers())) {
-                continue;
-            }
-
-            String name = Utils.getMemberName(m);
-            name = name.substring(0, 1).toUpperCase() + name.substring(1);
-
-            // Default for SDK bindings.
-            final AnnotationInfo info = new AnnotationInfo(name,
-                    AnnotationInfo.ExceptionMode.NSRESULT,
-                    AnnotationInfo.CallingThread.ANY,
-                    AnnotationInfo.DispatchTarget.CURRENT);
-            final AnnotatableEntity entity = new AnnotatableEntity(m, info);
-
-            if (m instanceof Constructor) {
-                generateConstructor(entity);
-            } else if (m instanceof Method) {
-                generateMethod(entity);
-            } else if (m instanceof Field) {
-                generateField(entity);
-            } else {
-                throw new IllegalArgumentException(
-                        "expected member to be Constructor, Method, or Field");
-            }
-        }
-    }
-
     public void generateClasses(final ClassWithOptions[] classes) {
         if (classes.length == 0) {
             return;

build/annotationProcessors/SDKProcessor.java

@@ -4,6 +4,75 @@
 
 package org.mozilla.gecko.annotationProcessors;
 
+/**
+ * Generate C++ bindings for SDK classes using a config file.
+ *
+ * java SDKProcessor <sdkjar> <configfile> <outdir> <fileprefix> <max-sdk-version>
+ *
+ * <sdkjar>: jar file containing the SDK classes (e.g. android.jar)
+ * <configfile>: config file for generating bindings
+ * <outdir>: output directory for generated binding files
+ * <fileprefix>: prefix used for generated binding files
+ * <max-sdk-version>: SDK version for generated class members (bindings will not be
+ *                     generated for members with SDK versions higher than max-sdk-version)
+ *
+ * The config file is a text file following the .ini format:
+ *
+ * ; comment
+ * [section1]
+ * property = value
+ *
+ * # comment
+ * [section2]
+ * property = value
+ *
+ * Each section specifies a qualified SDK class. Each property specifies a
+ * member of that class. The class and/or the property may specify options
+ * found in the WrapForJNI annotation. For example,
+ *
+ * # Generate bindings for Bundle using default options:
+ * [android.os.Bundle]
+ *
+ * # Generate bindings for Bundle using class options:
+ * [android.os.Bundle = exceptionMode:nsresult]
+ *
+ * # Generate bindings for Bundle using method options:
+ * [android.os.Bundle]
+ * putInt = stubName:PutInteger
+ *
+ * # Generate bindings for Bundle using class options with method override:
+ * # (note that all options are overriden at the same time.)
+ * [android.os.Bundle = exceptionMode:nsresult]
+ * # putInt will have stubName "PutInteger", and exceptionMode of "abort"
+ * putInt = stubName:PutInteger
+ * # putChar will have stubName "PutCharacter", and exceptionMode of "nsresult"
+ * putChar = stubName:PutCharacter, exceptionMode:nsresult
+ *
+ * # Overloded methods can be specified using its signature
+ * [android.os.Bundle]
+ * # Skip the copy constructor
+ * <init>(Landroid/os/Bundle;)V = skip:true
+ *
+ * # Generic member types can be specified
+ * [android.view.KeyEvent = skip:true]
+ * # Skip everything except fields
+ * <field> = skip:false
+ *
+ * # Skip everything except putInt and putChar
+ * [android.os.Bundle = skip:true]
+ * putInt = skip:false
+ * putChar =
+ *
+ * # Avoid conflicts in native bindings
+ * [android.os.Bundle]
+ * # Bundle(PersistableBundle) native binding can conflict with Bundle(ClassLoader)
+ * <init>(Landroid/os/PersistableBundle;)V = stubName:NewFromPersistableBundle
+ *
+ * # Generate a getter instead of a literal for certain runtime constants
+ * [android.os.Build$VERSION = skip:true]
+ * SDK_INT = noLiteral:true
+ */
+
 import com.android.tools.lint.checks.ApiLookup;
 import com.android.tools.lint.LintCliClient;
 
@@ -13,17 +82,17 @@ import org.mozilla.gecko.annotationProcessors.classloader.IterableJarLoadingURLC
 import org.mozilla.gecko.annotationProcessors.utils.GeneratableElementIterator;
 import org.mozilla.gecko.annotationProcessors.utils.Utils;
 
+import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.FileReader;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
-import java.util.Properties;
-import java.util.Scanner;
-import java.util.Vector;
+import java.util.Locale;
 import java.net.URL;
 import java.net.URLClassLoader;
 
@@ -45,17 +114,126 @@ public class SDKProcessor {
     private static ApiLookup sApiLookup;
     private static int sMaxSdkVersion;
 
+    private static class ParseException extends Exception {
+        public ParseException(final String message) {
+            super(message);
+        }
+    }
+
+    private static class ClassInfo {
+        public final String name;
+
+        // Map constructor/field/method signature to a set of annotation values.
+        private final HashMap<String, String> mAnnotations = new HashMap<>();
+        // Default set of annotation values to use.
+        private final String mDefaultAnnotation;
+
+        public ClassInfo(final String text) {
+            final String[] mapping = text.split("=", 2);
+            name = mapping[0].trim();
+            mDefaultAnnotation = mapping.length > 1 ? mapping[1].trim() : null;
+        }
+
+        public void addAnnotation(final String text) throws ParseException {
+            final String[] mapping = text.split("=", 2);
+            final String prop = mapping[0].trim();
+            if (prop.isEmpty()) {
+                throw new ParseException("Missing member name: " + text);
+            }
+            if (mapping.length < 2) {
+                throw new ParseException("Missing equal sign: " + text);
+            }
+            if (mAnnotations.get(prop) != null) {
+                throw new ParseException("Already has member: " + prop);
+            }
+            mAnnotations.put(prop, mapping[1].trim());
+        }
+
+        public AnnotationInfo getAnnotationInfo(final Member member) throws ParseException {
+            String stubName = Utils.getNativeName(member);
+            AnnotationInfo.ExceptionMode mode = AnnotationInfo.ExceptionMode.ABORT;
+            AnnotationInfo.CallingThread thread = AnnotationInfo.CallingThread.ANY;
+            AnnotationInfo.DispatchTarget target = AnnotationInfo.DispatchTarget.CURRENT;
+            boolean noLiteral = false;
+            boolean isGeneric = false;
+
+            final String name = Utils.getMemberName(member);
+            String annotation = mAnnotations.get(
+                    name + (member instanceof Field ? ":" : "") + Utils.getSignature(member));
+            if (annotation == null) {
+                // Match name without signature
+                annotation = mAnnotations.get(name);
+            }
+            if (annotation == null) {
+                // Match <constructor>, <field>, <method>
+                annotation = mAnnotations.get("<" + member.getClass().getSimpleName()
+                                                          .toLowerCase(Locale.ROOT) + '>');
+                isGeneric = true;
+            }
+            if (annotation == null) {
+                // Fallback on class options, if any.
+                annotation = mDefaultAnnotation;
+            }
+            if (annotation == null || annotation.isEmpty()) {
+                return new AnnotationInfo(stubName, mode, thread, target, noLiteral);
+            }
+
+            final String[] elements = annotation.split(",");
+            for (final String element : elements) {
+                final String[] pair = element.split(":", 2);
+                if (pair.length < 2) {
+                    throw new ParseException("Missing option value: " + element);
+                }
+                final String pairName = pair[0].trim();
+                final String pairValue = pair[1].trim();
+                switch (pairName) {
+                    case "skip":
+                        if (Boolean.valueOf(pairValue)) {
+                            // Return null to signal skipping current method.
+                            return null;
+                        }
+                        break;
+                    case "stubName":
+                        if (isGeneric) {
+                            // Prevent specifying stubName for class options.
+                            throw new ParseException("stubName doesn't make sense here: " +
+                                                     pairValue);
+                        }
+                        stubName = pairValue;
+                        break;
+                    case "exceptionMode":
+                        mode = Utils.getEnumValue(AnnotationInfo.ExceptionMode.class,
+                                                  pairValue);
+                        break;
+                    case "calledFrom":
+                        thread = Utils.getEnumValue(AnnotationInfo.CallingThread.class,
+                                                    pairValue);
+                        break;
+                    case "dispatchTo":
+                        target = Utils.getEnumValue(AnnotationInfo.DispatchTarget.class,
+                                                    pairValue);
+                        break;
+                    case "noLiteral":
+                        noLiteral = Boolean.valueOf(pairValue);
+                        break;
+                    default:
+                        throw new ParseException("Unknown option: " + pairName);
+                }
+            }
+            return new AnnotationInfo(stubName, mode, thread, target, noLiteral);
+        }
+    }
+
     public static void main(String[] args) throws Exception {
         // We expect a list of jars on the commandline. If missing, whinge about it.
         if (args.length < 5) {
-            System.err.println("Usage: java SDKProcessor sdkjar classlistfile outdir fileprefix max-sdk-version");
+            System.err.println("Usage: java SDKProcessor sdkjar configfile outdir fileprefix max-sdk-version");
             System.exit(1);
         }
 
         System.out.println("Processing platform bindings...");
 
         String sdkJar = args[0];
-        Vector classes = getClassList(args[1]);
         String outdir = args[2];
         String generatedFilePrefix = args[3];
         sMaxSdkVersion = Integer.parseInt(args[4]);
@@ -100,13 +278,25 @@ public class SDKProcessor {
             throw new RuntimeException(e.toString());
         }
 
-        for (Iterator<String> i = classes.iterator(); i.hasNext(); ) {
-            String className = i.next();
-            System.out.println("Looking up: " + className);
-
-            generateClass(Class.forName(className, true, loader),
-                          implementationFile,
-                          headerFile);
+        try {
+            final ClassInfo[] classes = getClassList(args[1]);
+            for (final ClassInfo cls : classes) {
+                System.out.println("Looking up: " + cls.name);
+                generateClass(Class.forName(cls.name, true, loader),
+                              cls,
+                              implementationFile,
+                              headerFile);
+            }
+        } catch (final IllegalStateException|IOException|ParseException e) {
+            System.err.println("***");
+            System.err.println("*** Error parsing config file: " + args[1]);
+            System.err.println("*** " + e);
+            System.err.println("***");
+            if (e.getCause() != null) {
+                e.getCause().printStackTrace(System.err);
+            }
+            System.exit(1);
+            return;
         }
 
         implementationFile.append(
@@ -148,7 +338,12 @@ public class SDKProcessor {
         });
 
         ArrayList<Member> list = new ArrayList<>();
-        for (Member m : members) {
+        for (final Member m : members) {
+            if (m.getDeclaringClass() == Object.class) {
+                // Skip methods from Object.
+                continue;
+            }
+
             // Sometimes (e.g. Bundle) has methods that moved to/from a superclass in a later SDK
             // version, so we check for both classes and see if we can find a minimum SDK version.
             int version = getAPIVersion(cls, m);
@@ -157,8 +352,9 @@ public class SDKProcessor {
                 version = version2;
             }
             if (version > sMaxSdkVersion) {
-                System.out.println("Skipping " + m.getDeclaringClass().getName() + "." + m.getName() +
-                    ", version " + version + " > " + sMaxSdkVersion);
+                System.out.println("Skipping " + m.getDeclaringClass().getName() + "." +
+                                   Utils.getMemberName(m) + ", version " + version + " > " +
+                                   sMaxSdkVersion);
                 continue;
             }
 
@@ -169,8 +365,8 @@ public class SDKProcessor {
                 if (m instanceof Field && !m.equals(cls.getField(m.getName()))) {
                     // m is a field in a superclass that has been hidden by
                     // a field with the same name in a subclass.
-                    System.out.println("Skipping " + m.getName() +
-                                       " from " + m.getDeclaringClass());
+                    System.out.println("Skipping " + Utils.getMemberName(m) +
+                                       " from " + m.getDeclaringClass().getName());
                     continue;
                 }
             } catch (final NoSuchFieldException e) {
@@ -182,37 +378,99 @@ public class SDKProcessor {
         return list.toArray(new Member[list.size()]);
     }
 
+    private static void generateMembers(CodeGenerator generator, ClassInfo clsInfo,
+                                        Member[] members) throws ParseException {
+        for (Member m : members) {
+            if (!Modifier.isPublic(m.getModifiers())) {
+                continue;
+            }
+
+            // Default for SDK bindings.
+            final AnnotationInfo info = clsInfo.getAnnotationInfo(m);
+            if (info == null) {
+                // Skip this member.
+                continue;
+            }
+            final AnnotatableEntity entity = new AnnotatableEntity(m, info);
+
+            if (m instanceof Constructor) {
+                generator.generateConstructor(entity);
+            } else if (m instanceof Method) {
+                generator.generateMethod(entity);
+            } else if (m instanceof Field) {
+                generator.generateField(entity);
+            } else {
+                throw new IllegalArgumentException(
+                        "expected member to be Constructor, Method, or Field");
+            }
+        }
+    }
+
     private static void generateClass(Class<?> clazz,
+                                      ClassInfo clsInfo,
                                       StringBuilder implementationFile,
-                                      StringBuilder headerFile) {
+                                      StringBuilder headerFile) throws ParseException {
         String generatedName = clazz.getSimpleName();
 
         CodeGenerator generator = new CodeGenerator(new ClassWithOptions(clazz, generatedName));
 
-        generator.generateMembers(sortAndFilterMembers(clazz, clazz.getConstructors()));
-        generator.generateMembers(sortAndFilterMembers(clazz, clazz.getMethods()));
-        generator.generateMembers(sortAndFilterMembers(clazz, clazz.getFields()));
+        generateMembers(generator, clsInfo,
+                        sortAndFilterMembers(clazz, clazz.getConstructors()));
+        generateMembers(generator, clsInfo, sortAndFilterMembers(clazz, clazz.getMethods()));
+        generateMembers(generator, clsInfo, sortAndFilterMembers(clazz, clazz.getFields()));
 
         headerFile.append(generator.getHeaderFileContents());
         implementationFile.append(generator.getWrapperFileContents());
     }
 
-    private static Vector<String> getClassList(String path) {
-        Scanner scanner = null;
-        try {
-            scanner = new Scanner(new FileInputStream(path));
+    private static ClassInfo[] getClassList(BufferedReader reader)
+            throws ParseException, IOException {
+        final ArrayList<ClassInfo> classes = new ArrayList<>();
+        ClassInfo currentClass = null;
+        String line;
 
-            Vector lines = new Vector();
-            while (scanner.hasNextLine()) {
-                lines.add(scanner.nextLine());
+        while ((line = reader.readLine()) != null) {
+            line = line.trim();
+            if (line.isEmpty()) {
+                continue;
             }
-            return lines;
-        } catch (Exception e) {
-            System.out.println(e.toString());
-            return null;
+            switch (line.charAt(0)) {
+                case ';':
+                case '#':
+                    // Comment
+                    continue;
+                case '[':
+                    // New section
+                    if (line.charAt(line.length() - 1) != ']') {
+                        throw new ParseException("Missing trailing ']': " + line);
+                    }
+                    currentClass = new ClassInfo(line.substring(1, line.length() - 1));
+                    classes.add(currentClass);
+                    break;
+                default:
+                    // New mapping
+                    if (currentClass == null) {
+                        throw new ParseException("Missing class: " + line);
+                    }
+                    currentClass.addAnnotation(line);
+                    break;
+            }
+        }
+        if (classes.isEmpty()) {
+            throw new ParseException("No class found in config file");
+        }
+        return classes.toArray(new ClassInfo[classes.size()]);
+    }
+
+    private static ClassInfo[] getClassList(final String path)
+            throws ParseException, IOException {
+        BufferedReader reader = null;
+        try {
+            reader = new BufferedReader(new FileReader(path));
+            return getClassList(reader);
         } finally {
-            if (scanner != null) {
-                scanner.close();
+            if (reader != null) {
+                reader.close();
             }
         }
     }

build/annotationProcessors/utils/GeneratableElementIterator.java

@@ -121,32 +121,6 @@ public class GeneratableElementIterator implements Iterator<AnnotatableEntity> {
         return ret;
     }
 
-    private static <T extends Enum<T>> T getEnumValue(Class<T> type, String name)
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException {
-        try {
-            return Enum.valueOf(type, name.toUpperCase());
-
-        } catch (IllegalArgumentException e) {
-            Object[] values = (Object[]) type.getDeclaredMethod("values").invoke(null);
-            StringBuilder names = new StringBuilder();
-
-            for (int i = 0; i < values.length; i++) {
-                if (i != 0) {
-                    names.append(", ");
-                }
-                names.append(values[i].toString().toLowerCase());
-            }
-
-            System.err.println("***");
-            System.err.println("*** Invalid value \"" + name + "\" for " + type.getSimpleName());
-            System.err.println("*** Specify one of " + names.toString());
-            System.err.println("***");
-            e.printStackTrace(System.err);
-            System.exit(6);
-            return null;
-        }
-    }
-
     private AnnotationInfo buildAnnotationInfo(AnnotatedElement element, Annotation annotation) {
         Class<? extends Annotation> annotationType = annotation.annotationType();
         final String annotationTypeName = annotationType.getName();
@@ -175,19 +149,19 @@ public class GeneratableElementIterator implements Iterator<AnnotatableEntity> {
 
             final Method exceptionModeMethod = annotationType.getDeclaredMethod("exceptionMode");
             exceptionModeMethod.setAccessible(true);
-            exceptionMode = getEnumValue(
+            exceptionMode = Utils.getEnumValue(
                     AnnotationInfo.ExceptionMode.class,
                     (String) exceptionModeMethod.invoke(annotation));
 
             final Method calledFromMethod = annotationType.getDeclaredMethod("calledFrom");
             calledFromMethod.setAccessible(true);
-            callingThread = getEnumValue(
+            callingThread = Utils.getEnumValue(
                     AnnotationInfo.CallingThread.class,
                     (String) calledFromMethod.invoke(annotation));
 
             final Method dispatchToMethod = annotationType.getDeclaredMethod("dispatchTo");
             dispatchToMethod.setAccessible(true);
-            dispatchTarget = getEnumValue(
+            dispatchTarget = Utils.getEnumValue(
                     AnnotationInfo.DispatchTarget.class,
                     (String) dispatchToMethod.invoke(annotation));
 

build/annotationProcessors/utils/Utils.java

@@ -9,11 +9,13 @@ import org.mozilla.gecko.annotationProcessors.AnnotationInfo;
 import java.lang.reflect.AnnotatedElement;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Member;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.nio.ByteBuffer;
 import java.util.HashMap;
+import java.util.Locale;
 
 /**
  * A collection of utility methods used by CodeGenerator. Largely used for translating types.
@@ -219,7 +221,7 @@ public class Utils {
      */
     public static String getNativeName(Member member) {
         final String name = getMemberName(member);
-        return name.substring(0, 1).toUpperCase() + name.substring(1);
+        return name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1);
     }
 
     /**
@@ -230,7 +232,7 @@ public class Utils {
      */
     public static String getNativeName(Class<?> clz) {
         final String name = clz.getName();
-        return name.substring(0, 1).toUpperCase() + name.substring(1);
+        return name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1);
     }
 
     /**
@@ -285,4 +287,44 @@ public class Utils {
     public static boolean isFinal(final Member member) {
         return Modifier.isFinal(member.getModifiers());
     }
+
+    /**
+     * Return an enum value with the given name.
+     *
+     * @param type Enum class type.
+     * @param name Enum value name.
+     * @return Enum value with the given name.
+     */
+    public static <T extends Enum<T>> T getEnumValue(Class<T> type, String name) {
+        try {
+            return Enum.valueOf(type, name.toUpperCase(Locale.ROOT));
+
+        } catch (IllegalArgumentException e) {
+            final Object[] values;
+            try {
+                values = (Object[]) type.getDeclaredMethod("values").invoke(null);
+            } catch (final NoSuchMethodException |
+                           IllegalAccessException |
+                           InvocationTargetException exception) {
+                throw new RuntimeException("Cannot access enum: " + type, exception);
+            }
+
+            StringBuilder names = new StringBuilder();
+
+            for (int i = 0; i < values.length; i++) {
+                if (i != 0) {
+                    names.append(", ");
+                }
+                names.append(values[i].toString().toLowerCase(Locale.ROOT));
+            }
+
+            System.err.println("***");
+            System.err.println("*** Invalid value \"" + name + "\" for " + type.getSimpleName());
+            System.err.println("*** Specify one of " + names.toString());
+            System.err.println("***");
+            e.printStackTrace(System.err);
+            System.exit(1);
+            return null;
+        }
+    }
 }