[Python APIView] Fix type parsing regression in Python 3.10 (#2479)

* Fix #2478.

* Use Python 3.10 in the CI.

* Fix #752.

* Refine typehint mismatch message.

* Wrap kwargs in "Optional[]" so as to match the typehint representation.

* Bump version to 0.2.8.

* Update packages/python-packages/api-stub-generator/apistub/nodes/_docstring_parser.py

Co-authored-by: Johan Stenberg (MSFT) <johan.stenberg@microsoft.com>

* Update packages/python-packages/api-stub-generator/CHANGELOG.md

Co-authored-by: Charles Lowell <10964656+chlowell@users.noreply.github.com>

* Fix #2522.

Co-authored-by: Johan Stenberg (MSFT) <johan.stenberg@microsoft.com>
Co-authored-by: Charles Lowell <10964656+chlowell@users.noreply.github.com>

.azure-pipelines/apiview.yml

@@ -157,9 +157,9 @@ stages:
         steps:
 
           - task: UsePythonVersion@0
-            displayName: 'Use Python 3.6'
+            displayName: 'Use Python 3.10'
             inputs:
-              versionSpec: 3.6
+              versionSpec: 3.10
 
           - task: DotNetCoreInstaller@2
             displayName: 'Use .NET Core sdk $(DotNetCoreVersion)'

packages/python-packages/api-stub-generator/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release History
 
+## Version 0.2.8 (Unreleased)
+Kwargs that were previously displayed as "type = ..." will now
+  be displayed as "Optional[type] = ..." to align with syntax
+  from Python type hints.
+Fixed issue where variadic arguments appeared a singular argument.
+Fixes issue where default values were no longer displayed.
+
 ## Version 0.2.7 (Unreleased)
 Updated version to regenerate all reviews using Python 3.9
 

packages/python-packages/api-stub-generator/apistub/_apiview.py

@@ -19,7 +19,6 @@ TYPE_OR_SEPERATOR = " or "
 
 # Lint warnings
 SOURCE_LINK_NOT_AVAILABLE = "Source definition link is not available for [{0}]. Please check and ensure type is fully qualified name in docstring"
-RETURN_TYPE_MISMATCH = "Return type in type hint is not matching return type in docstring"
 
 
 class ApiView:

packages/python-packages/api-stub-generator/apistub/_version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-VERSION = "0.2.7"
+VERSION = "0.2.8"

packages/python-packages/api-stub-generator/apistub/nodes/_base_node.py

@@ -1,4 +1,5 @@
 from inspect import Parameter
+from typing import Optional
 
 class NodeEntityBase:
     """This is the base class for all node types
@@ -54,6 +55,10 @@ def get_qualified_name(obj, namespace):
     name = str(obj)
     if hasattr(obj, "__name__"):
         name = getattr(obj, "__name__")
+        # workaround because typing.Optional __name__ is just Optional in Python 3.10
+        # but not in previous versions
+        if name == "Optional":
+            name = str(obj)
     elif hasattr(obj, "__qualname__"):
         name = getattr(obj, "__qualname__")
 

packages/python-packages/api-stub-generator/apistub/nodes/_docstring_parser.py

@@ -100,7 +100,11 @@ class DocstringParser:
             self.pos_args[arg.argname] = arg
         elif keyword == "keyword":
             # show kwarg is optional by setting default to "..."
+            # also wrap the type in Optional[] so it aligns with
+            # optionals identified in type hints.
             arg.default = "..."
+            if arg.argtype and not arg.argtype.startswith("Optional["):
+                arg.argtype = f"Optional[{arg.argtype}]"
             self.kw_args[arg.argname] = arg
         else:
             logging.error(f"Unexpected keyword: {keyword}")

packages/python-packages/api-stub-generator/apistub/nodes/_function_node.py

@@ -123,17 +123,27 @@ class FunctionNode(NodeEntityBase):
         # Add all keyword only args here temporarily until docstring is parsed
         # This is to handle the scenario for keyword arg typehint (py3 style is present in signature itself)
         self.kw_args = OrderedDict()
-        for argname in params:
-            arg = ArgType(argname, get_qualified_name(params[argname].annotation, self.namespace), "", self)
+        for argname, argvalues in params.items():
+            arg = ArgType(argname, get_qualified_name(argvalues.annotation, self.namespace), "", self)
             # set default value if available
-            if params[argname].default != Parameter.empty:
-                arg.default = str(params[argname].default)
+            if argvalues.default != Parameter.empty:
+                arg.default = str(argvalues.default)
+
             # Store handle to kwarg object to replace it later
-            if params[argname].kind == Parameter.VAR_KEYWORD:
+            if argvalues.kind == Parameter.VAR_KEYWORD:
                 arg.argname = f"**{argname}"
 
-            if params[argname].kind == Parameter.KEYWORD_ONLY:
+            if argvalues.kind == Parameter.KEYWORD_ONLY:
+                # Keyword-only args with "None" default are displayed as "..."
+                # to match logic in docstring parsing
+                if arg.default == "None":
+                    arg.default = "..."
                 self.kw_args[arg.argname] = arg
+            elif argvalues.kind == Parameter.VAR_POSITIONAL:
+                # to work with docstring parsing, the key must
+                # not have the * in it.
+                arg.argname = f"*{argname}"
+                self.args[argname] = arg
             else:
                 self.args[arg.argname] = arg
 
@@ -213,7 +223,7 @@ class FunctionNode(NodeEntityBase):
                 if not docstring_match:
                     continue
                 signature_arg.argtype = docstring_match.argtype or signature_arg.argtype
-                signature_arg.default = docstring_match.default or signature_arg.default
+                signature_arg.default = signature_arg.default if signature_arg.default is not None else docstring_match.default
 
             # Update keyword argument metadata from the docstring; otherwise, stick with
             # what was parsed from the signature.
@@ -224,7 +234,7 @@ class FunctionNode(NodeEntityBase):
                     continue
                 remaining_docstring_kwargs.remove(argname)
                 kw_arg.argtype = docstring_match.argtype or kw_arg.argtype
-                kw_arg.default = docstring_match.default or kw_arg.default
+                kw_arg.default = kw_arg.default if kw_arg.default is not None else docstring_match.default
             
             # ensure any kwargs described only in the docstrings are added
             for argname in remaining_docstring_kwargs:
@@ -264,7 +274,7 @@ class FunctionNode(NodeEntityBase):
             long_ret_type = self.return_type
             if long_ret_type != type_hint_ret_type and short_return_type != type_hint_ret_type:
                 logging.info("Long type: {0}, Short type: {1}, Type hint return type: {2}".format(long_ret_type, short_return_type, type_hint_ret_type))
-                error_message = "Return type in type hint is not matching return type in docstring"
+                error_message = "The return type is described in both a type hint and docstring, but they do not match."
                 self.add_error(error_message)
 
 

packages/python-packages/api-stub-generator/tests/docstring_parser_test.py

@@ -172,9 +172,9 @@ class TestDocstringParser:
     def test_docstring_multi_complex_type(self):
         self._test_variable_type(docstring_multi_complex_type, {
             "documents": "list[str] or list[~azure.ai.textanalytics.DetectLanguageInput] or list[dict[str, str]]",
-            "country_hint": "str",
-            "model_version": "str",
-            "show_stats": "bool"
+            "country_hint": "Optional[str]",
+            "model_version": "Optional[str]",
+            "show_stats": "Optional[bool]"
         })
 
     def test_docstring_param_type_private(self):

packages/python-packages/api-stub-generator/tests/function_parsing_test.py

@@ -0,0 +1,71 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+from apistub.nodes import ClassNode, FunctionNode
+
+from typing import Optional, Any
+
+class TestClass:
+    """ Function parsing tests."""
+
+    def with_optional_typehint(self, *, description: Optional[str] = None):
+        self.description = description
+
+    def with_optional_docstring(self, *, description = None):
+        """ With docstring
+        :keyword description: A description
+        :paramtype description: str
+        """
+        self.description = description
+
+    def with_variadic_python3_typehint(self, *vars: str, **kwargs: "Any") -> None:
+        return None
+
+    def with_variadic_python2_typehint(self, *vars, **kwargs):
+        # type: (*str, **Any) -> None
+        """ With docstring
+        :param vars: Variadic argument
+        :type vars: str
+        """
+        return None
+
+    def with_default_values(self, foo="1", *, bar="2", baz=None):
+        return None
+
+
+class TestFunctionParsing:
+    
+    def test_optional_typehint(self):
+        func_node = FunctionNode("test", None, TestClass.with_optional_typehint, "test")
+        arg = func_node.args["description"]
+        assert arg.argtype == "typing.Optional[str]"
+        assert arg.default == "..."
+
+    def test_optional_docstring(self):
+        func_node = FunctionNode("test", None, TestClass.with_optional_docstring, "test")
+        arg = func_node.args["description"]
+        assert arg.argtype == "str"
+        assert arg.default == "..."
+
+    def test_variadic_typehints(self):
+        func_node = FunctionNode("test", None, TestClass.with_variadic_python3_typehint, "test")
+        arg = func_node.args["vars"]
+        assert arg.argname == "*vars"
+        assert arg.argtype == "str"
+        assert arg.default == ""
+
+        func_node = FunctionNode("test", None, TestClass.with_variadic_python2_typehint, "test")
+        arg = func_node.args["vars"]
+        assert arg.argname == "*vars"
+        # the type annotation comes ONLY from the docstring. The Python2 type hint is not used!
+        assert arg.argtype == "str"
+        assert arg.default == ""
+
+    def test_default_values(self):
+        func_node = FunctionNode("test", None, TestClass.with_default_values, "test")
+        assert func_node.args["foo"].default == "1"
+        assert func_node.kw_args["bar"].default == "2"
+        assert func_node.kw_args["baz"].default == "..."

packages/python-packages/api-stub-generator/tox.ini

@@ -1,6 +1,6 @@
 [tox]
 # note that this envlist is the default set of environments that will run if a target environment is not selected.
-envlist = py36
+envlist = py310
 
 [testenv]
 deps = pytest