[Python APIView] add script to test all SDK packages (#3163)

* Add initial results.

* Update script.

* Fix issue with type parsing.

eng/scripts/python_sdk_report.py

@@ -0,0 +1,82 @@
+#!/usr/bin/python3
+
+"""
+python_sdk_report.py
+Generate APIView for all SDKs in the azure-sdk-for-python repo and report on any failures.
+"""
+
+import glob
+import json
+import os
+import re
+import sys
+from typing import Optional
+
+from apistub import StubGenerator
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
+
+PACKAGE_NAME_RE = re.compile(r"sdk\\([a-z]+)\\([a-z\-]+)\\setup.py")
+
+SKIP_PACKAGES = [
+    "core:azure",
+    "core:azure-mgmt",
+    "monitor:azure-monitor",
+    "storage:azure-storage"
+]
+
+class _Result:
+    def __init__(self, *, service_dir: str, package_name: str, success: bool, error: Optional[str]):
+        self.service_dir = service_dir
+        self.package_name = package_name
+        self.success = success
+        self.error = error
+
+if __name__ == '__main__':
+    warning_color = '\033[91m'
+    end_color = '\033[0m'
+
+    stub_gen_path = os.path.join(ROOT, 'packages', 'python-packages', 'api-stub-generator')
+    changelog_path = os.path.join(stub_gen_path, "CHANGELOG.md")
+    version_path = os.path.join(stub_gen_path, 'apistub', '_version.py')
+
+    args =  sys.argv
+    if len(args) != 2:
+        print("usage: python python_sdk_report.py <PYTHON SDK REPO ROOT>")
+        sys.exit(1)
+    
+    python_sdk_root = args[1]
+    print(f"Python SDK Root: {python_sdk_root}")
+
+    results = {}
+    for path in glob.glob(os.path.join(python_sdk_root, "sdk", "**", "**", "setup.py")):
+        package_path = os.path.split(path)[0]
+        try:
+            (service_dir, package_name) = PACKAGE_NAME_RE.findall(path)[0]
+        except:
+            print(f"Couldn't parse: {path}")
+            continue
+
+        if f"{service_dir}:{package_name}" in SKIP_PACKAGES:
+            continue
+
+        print(f"Parsing {service_dir}/{package_name}...")
+        if service_dir not in results:
+            results[service_dir] = []
+        try:
+            _ = StubGenerator(pkg_path=package_path, skip_pylint=True).generate_tokens()
+            success = True
+            error = None
+        except Exception as err:
+            success = False
+            error = str(err)
+        results[service_dir].append(_Result(
+            service_dir=service_dir,
+            package_name=package_name,
+            success=success,
+            error=error
+        ))
+    filename = "stubgen_report.json"
+    print(f"Saving results to {filename}...")
+    with open(filename, "w") as outfile:
+        outfile.write(json.dumps(results, indent=4))

packages/python-packages/api-stub-generator/apistub/__init__.py

@@ -23,7 +23,6 @@ __all__ = [
 
 
 def console_entry_point():
-    from apistub.nodes._pylint_parser import PylintParser
     print("Running api-stub-generator version {}".format(__version__))
     stub_generator = StubGenerator()
     apiview = stub_generator.generate_tokens()

packages/python-packages/api-stub-generator/apistub/_apiview.py

@@ -14,7 +14,7 @@ from ._metadata_map import MetadataMap
 JSON_FIELDS = ["Name", "Version", "VersionString", "Navigation", "Tokens", "Diagnostics", "PackageName", "Language"]
 
 HEADER_TEXT = "# Package is parsed using api-stub-generator(version:{0}), Python version: {1}".format(VERSION, platform.python_version())
-TYPE_NAME_REGEX = re.compile(r"(~?[a-zA-Z\\d._]+)")
+TYPE_NAME_REGEX = re.compile(r"(~?[a-zA-Z\d._]+)")
 TYPE_OR_SEPARATOR = " or "
 
 
@@ -181,6 +181,7 @@ class ApiView:
     def _add_type_token(self, type_name, line_id):
         # parse to get individual type name
         logging.debug("Generating tokens for type {}".format(type_name))
+
         types = re.search(TYPE_NAME_REGEX, type_name)
         if types:
             # Generate token for the prefix before internal type

packages/python-packages/api-stub-generator/apistub/_stub_generator.py

@@ -31,9 +31,10 @@ logging.getLogger().setLevel(logging.ERROR)
 
 
 class StubGenerator:
-    def __init__(self, args=None):
+    def __init__(self, **kwargs):
         from .nodes import PylintParser
-        if not args:
+        self._kwargs = kwargs
+        if not kwargs:
             parser = argparse.ArgumentParser(
                 description="Parses a Python package and generates a JSON token file for consumption by the APIView tool."
             )
@@ -69,24 +70,49 @@ class StubGenerator:
                 "--source-url",
                 help=("URL to the pull request URL that contains the source used to generate this APIView.")
             )
-            args = parser.parse_args()
+            parser.add_argument(
+                "--skip-pylint",
+                help=("Skips running pylint on the package to obtain diagnostics."),
+                default=False,
+                action="store_true"
+            )
+            self._args = parser.parse_args()
 
-        if not os.path.exists(args.pkg_path):
-            logging.error("Package path [{}] is invalid".format(args.pkg_path))
+        pkg_path = self._parse_arg("pkg_path")
+        temp_path = self._parse_arg("temp_path") or tempfile.gettempdir()
+        out_path = self._parse_arg("out_path")
+        mapping_path = self._parse_arg("mapping_path")
+        verbose = self._parse_arg("verbose")
+        filter_namespace = self._parse_arg("filter_namespace")
+        source_url = self._parse_arg("source_url")
+        skip_pylint = self._parse_arg("skip_pylint")
+
+        if not os.path.exists(pkg_path):
+            logging.error("Package path [{}] is invalid".format(pkg_path))
             exit(1)
-        elif not os.path.exists(args.temp_path):
-            logging.error("Temp path [{0}] is invalid".format(args.temp_path))
+        elif not os.path.exists(temp_path):
+            logging.error("Temp path [{0}] is invalid".format(temp_path))
             exit(1)
 
-        self.pkg_path = args.pkg_path
-        self.temp_path = args.temp_path
-        self.out_path = args.out_path
-        self.source_url = args.source_url
-        self.mapping_path = args.mapping_path
-        self.filter_namespace = args.filter_namespace or ''
-        if args.verbose:
+        self.pkg_path = pkg_path
+        self.temp_path = temp_path
+        self.out_path = out_path
+        self.source_url = source_url
+        self.mapping_path = mapping_path
+        self.filter_namespace = filter_namespace or ''
+        if verbose:
             logging.getLogger().setLevel(logging.DEBUG)
-        PylintParser.parse(self.pkg_path)
+        if not skip_pylint:
+            PylintParser.parse(pkg_path)
+
+    def _parse_arg(self, name):
+        value = self._kwargs.get(name, None)
+        if not value:
+            try:
+                value = getattr(self._args, name, None)
+            except AttributeError:
+                value = None
+        return value
 
     def generate_tokens(self):
         # Extract package to temp directory if it is wheel or sdist

packages/python-packages/api-stub-generator/apistub/nodes/_class_node.py

@@ -299,14 +299,14 @@ class ClassNode(NodeEntityBase):
         # Add inherited base classes
         if self.base_class_names:
             apiview.add_punctuation("(")
-            self._generate_token_for_collection(self.base_class_names, apiview)
+            self._generate_tokens_for_collection(self.base_class_names, apiview)
             apiview.add_punctuation(")")
         apiview.add_punctuation(":")
 
         # Add any ABC implementation list
         if self.implements:
             apiview.add_keyword("implements", True, True)
-            self._generate_token_for_collection(self.implements, apiview)
+            self._generate_tokens_for_collection(self.implements, apiview)
         apiview.add_newline()
 
         # Generate token for child nodes
@@ -332,11 +332,11 @@ class ClassNode(NodeEntityBase):
         apiview.end_group()
 
 
-    def _generate_token_for_collection(self, values, apiview):
+    def _generate_tokens_for_collection(self, values, apiview):
         # Helper method to concatenate list of values and generate tokens
         list_len = len(values)
-        for index in range(list_len):
-            apiview.add_type(values[index], self.namespace_id)
+        for (idx, value) in enumerate(values):
+            apiview.add_type(value, self.namespace_id)
             # Add punctuation between types
-            if index < list_len - 1:
+            if idx < list_len - 1:
                 apiview.add_punctuation(",", False, True)

packages/python-packages/api-stub-generator/apistub/nodes/_pylint_parser.py

@@ -56,8 +56,9 @@ class PylintParser:
         from apistub import ApiView
 
         pkg_name = os.path.split(path)[-1]
+        rcfile_path = os.path.join(ApiView.get_root_path(), "pylintrc")
         logging.debug(f"APIView root path: {ApiView.get_root_path()}")
-        (pylint_stdout, pylint_stderr) = epylint.py_run(f"{path} -f json --load-plugins pylint_guidelines_checker", return_std=True)
+        (pylint_stdout, pylint_stderr) = epylint.py_run(f"{path} -f json --rcfile {rcfile_path}", return_std=True)
         stderr_str = pylint_stderr.read()
         # strip put stray, non-json lines from stdout
         stdout_lines = [x for x in pylint_stdout.readlines() if not x.startswith("Exception")]

packages/python-packages/api-stub-generator/apistubgen.py

@@ -1,3 +1,4 @@
+import logging
 import sys
 
 from apistub import console_entry_point
@@ -6,5 +7,6 @@ if __name__ == "__main__":
     try:
         console_entry_point()
         sys.exit(0)
-    except:
+    except Exception as err:
+        logging.error(err)
         sys.exit(1)

packages/python-packages/api-stub-generator/tests/_test_util.py

@@ -32,4 +32,4 @@ def _merge_lines(lines) -> str:
 
 
 def _check(actual, expected, client):
-    assert actual == expected, f"\n*******\nClient: {client.__name__}\nActual:   {actual}\nExpected: {expected}\n*******"    
+    assert actual.lstrip() == expected, f"\n*******\nClient: {client.__name__}\nActual:   {actual}\nExpected: {expected}\n*******"    

packages/python-packages/api-stub-generator/tests/apiview_test.py

@@ -9,17 +9,6 @@ from apistub.nodes import PylintParser
 import os
 import tempfile
 
-from ._test_util import _check, _render_string, _tokenize
-
-class StubGenTestArgs:
-    pkg_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'apistubgentest'))
-    temp_path = tempfile.gettempdir()
-    source_url = None
-    out_path = None
-    mapping_path = None
-    verbose = None
-    filter_namespace = None
-
 
 class TestApiView:
     def _count_newlines(self, apiview):
@@ -54,12 +43,22 @@ class TestApiView:
         assert self._count_newlines(apiview) == 3 # +1 for carriage return
 
     def test_api_view_diagnostic_warnings(self):
-        args = StubGenTestArgs()
-        stub_gen = StubGenerator(args=args)
+        pkg_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "apistubgentest"))
+        temp_path = tempfile.gettempdir()
+        stub_gen = StubGenerator(pkg_path=pkg_path, temp_path=temp_path)
         apiview = stub_gen.generate_tokens()
         # ensure we have only the expected diagnostics when testing apistubgentest
         unclaimed = PylintParser.get_unclaimed()
-        assert len(apiview.diagnostics) == 4
+        assert len(apiview.diagnostics) == 5
         # The "needs copyright header" error corresponds to a file, which isn't directly
         # represented in APIView
         assert len(unclaimed) == 1
+
+    def test_add_type(self):
+        apiview = ApiView()
+        apiview.tokens = []
+        apiview.add_type(type_name="a.b.c.1.2.3.MyType")
+        tokens = apiview.tokens
+        assert len(tokens) == 2
+        assert tokens[0].kind == TokenKind.TypeName
+        assert tokens[1].kind == TokenKind.Punctuation

packages/python-packages/api-stub-generator/tests/class_parsing_test.py

@@ -12,6 +12,7 @@ from apistubgentest.models import (
     PublicPrivateClass,
     RequiredKwargObject,
     SomeAwesomelyNamedObject,
+    SomeImplementationClass,
     SomethingWithDecorators,
     SomethingWithOverloads,
     SomethingWithProperties
@@ -153,3 +154,15 @@ class TestClassParsing:
             expect = expected[idx]
             _check(actual, expect, SomethingWithProperties)
 
+    def test_abstract_class(self):
+        class_node = ClassNode(name="SomeImplementationClass", namespace=f"apistubgentest.models.SomeImplementationClass", parent_node=None, obj=SomeImplementationClass, pkg_root_namespace=self.pkg_namespace)
+        actuals = _render_lines(_tokenize(class_node))
+        expected = [
+            "class apistubgentest.models.SomeImplementationClass(_SomeAbstractBase):",
+            "",
+            "def say_hello(self) -> str"
+        ]
+        for (idx, actual) in enumerate(actuals):
+            expect = expected[idx]
+            _check(actual, expect, SomethingWithProperties)
+        

packages/python-packages/apistubgentest/apistubgentest/models/__init__.py

@@ -17,13 +17,13 @@ from ._models import (
     PetEnumPy3MetaclassAlt,
     PublicCaseInsensitiveEnumMeta,
     PublicPrivateClass, RequiredKwargObject,
+    SomeImplementationClass,
     SomePoorlyNamedObject as SomeAwesomelyNamedObject,
     SomethingWithDecorators,
     SomethingWithOverloads,
     SomethingWithProperties
 )
 
-
 __all__ = (
     "DocstringClass",
     "FakeError",
@@ -36,6 +36,7 @@ __all__ = (
     "PublicPrivateClass",
     "RequiredKwargObject",
     "SomeAwesomelyNamedObject",
+    "SomeImplementationClass",
     "SomethingWithDecorators",
     "SomethingWithOverloads",
     "SomethingWithProperties"

packages/python-packages/apistubgentest/apistubgentest/models/_models.py

@@ -6,12 +6,12 @@
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 
+import abc
 from azure.core import CaseInsensitiveEnumMeta
 from collections.abc import Sequence
 from dataclasses import dataclass
 from enum import Enum, EnumMeta
 import functools
-from six import with_metaclass
 from typing import Any, overload, TypedDict, Union, Optional
 
 
@@ -237,3 +237,18 @@ class SomethingWithProperties:
         :rtype: Optional[str]
         """
         pass
+
+# pylint:disable=docstring-missing-rtype
+class _SomeAbstractBase(abc.ABC):
+    """ Some abstract base class. """
+
+    @property
+    @abc.abstractmethod
+    def say_hello(self) -> str:
+        """ A method to say hello. """
+        ...
+
+class SomeImplementationClass(_SomeAbstractBase):
+
+    def say_hello(self) -> str:
+        return "Hello!"