Add python 3.8 nuget generation (#587)

* Upgrade Flake8 3.7.9 and fix new rules linting

* Update grpcio tools to 1.26.0 for py38 package generation

* Separate V2 host and V3 host package generation

.flake8

@@ -4,4 +4,5 @@ exclude =
     .git, __pycache__, build, dist, .eggs, .github, .local, docs/,
     Samples, azure_functions_worker/protos/,
     azure_functions_worker/typing_inspect.py,
-    tests/unittests/test_typing_inspect.py
+    tests/unittests/test_typing_inspect.py,
+    .venv*, .env*, .vscode

.gitignore

@@ -79,11 +79,11 @@ celerybeat-schedule
 # SageMath parsed files
 *.sage.py
 
-# dotenv
-.env
+# dotenv (.env/.env36/.env37/.env38)
+.env*
 
-# virtualenv
-.venv
+# virtualenv (.venv/.venv36/.venv37/.venv38)
+.venv*
 venv/
 ENV/
 

azure-pipelines.yml

@@ -1,4 +1,4 @@
-name: 1.0.$(Date:yyyyMMdd)$(Rev:r)
+name: 1.x.$(Date:yyyyMMdd)$(Rev:r)
 
 trigger:
 - dev
@@ -17,6 +17,8 @@ jobs:
         pythonVersion: '3.6'
       Python37:
         pythonVersion: '3.7'
+      Python38:
+        pythonVersion: '3.8'
     maxParallel: 1
   steps:
   - task: UsePythonVersion@0
@@ -54,7 +56,7 @@ jobs:
 
 - job: Build_WINDOWS_X64
   dependsOn: 'Tests'
-  pool: 
+  pool:
     vmImage: 'vs2017-win2016'
   strategy:
     matrix:
@@ -62,6 +64,8 @@ jobs:
         pythonVersion: '3.6'
       Python37:
         pythonVersion: '3.7'
+      Python38:
+        pythonVersion: '3.8'
   steps:
   - template: pack/templates/win_env_gen.yml
     parameters:
@@ -70,12 +74,14 @@ jobs:
       artifactName: '$(pythonVersion)_WINDOWS_X64'
 - job: Build_WINDOWS_X86
   dependsOn: 'Tests'
-  pool: 
+  pool:
     vmImage: 'vs2017-win2016'
   strategy:
     matrix:
       Python37:
         pythonVersion: '3.7'
+      Python38:
+        pythonVersion: '3.8'
   steps:
   - template: pack/templates/win_env_gen.yml
     parameters:
@@ -84,7 +90,7 @@ jobs:
       artifactName: '$(pythonVersion)_WINDOWS_x86'
 - job: Build_LINUX_X64
   dependsOn: 'Tests'
-  pool: 
+  pool:
     vmImage: 'ubuntu-16.04'
   strategy:
     matrix:
@@ -92,6 +98,8 @@ jobs:
         pythonVersion: '3.6'
       Python37:
         pythonVersion: '3.7'
+      Python38:
+        pythonVersion: '3.8'
   steps:
   - template: pack/templates/nix_env_gen.yml
     parameters:
@@ -99,7 +107,7 @@ jobs:
       artifactName: '$(pythonVersion)_LINUX_X64'
 - job: Build_OSX_X64
   dependsOn: 'Tests'
-  pool: 
+  pool:
     vmImage: 'macOS-10.13'
   strategy:
     matrix:
@@ -107,6 +115,8 @@ jobs:
         pythonVersion: '3.6'
       Python37:
         pythonVersion: '3.7'
+      Python38:
+        pythonVersion: '3.8'
   steps:
   - template: pack/templates/nix_env_gen.yml
     parameters:
@@ -121,7 +131,22 @@ jobs:
              ]
   pool:
       vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      V2PythonWorker:
+        minorVersion: '0'
+        nuspecPath: 'pack\Microsoft.Azure.Functions.V2.PythonWorker.nuspec'
+      V3PythonWorker:
+        minorVersion: '1'
+        nuspecPath: 'pack\Microsoft.Azure.Functions.V3.PythonWorker.nuspec'
   steps:
+  - bash: |
+      MAJOR=$(echo $BUILD_BUILDNUMBER | cut -d '.' -f1)
+      MINOR=$(minorVersion)
+      PATCH=$(echo $BUILD_BUILDNUMBER | cut -d '.' -f3)
+      echo "##vso[task.setvariable variable=worker_version]$MAJOR.$MINOR.$PATCH"
+      echo "Packaging Python Worker Version $MAJOR.$MINOR.$PATCH"
+    displayName: "Generate Worker Version"
   - task: DownloadBuildArtifacts@0
     inputs:
       buildType: 'current'
@@ -130,11 +155,10 @@ jobs:
   - task: NuGetCommand@2
     inputs:
       command: pack
-      packagesToPack: 'pack\Microsoft.Azure.Functions.PythonWorker.nuspec'
+      packagesToPack: '$(nuspecPath)'
       versioningScheme: 'byEnvVar'
-      versionEnvVar: BUILD_BUILDNUMBER # Replaces version in nuspec
+      versionEnvVar: WORKER_VERSION # Replaces version in nuspec
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: 'PythonWorker'
-      

azure_functions_worker/functions.py

@@ -151,16 +151,16 @@ class Registry:
                 # typing_inspect.get_args() returns a flat list,
                 # so if the annotation was func.Out[typing.List[foo]],
                 # we need to reconstruct it.
-                if (isinstance(param_py_type, tuple) and
-                        typing_inspect.is_generic_type(param_py_type[0])):
+                if (isinstance(param_py_type, tuple)
+                   and typing_inspect.is_generic_type(param_py_type[0])):
 
                     param_py_type = operator.getitem(
                         param_py_type[0], *param_py_type[1:])
             else:
                 param_py_type = param_anno
 
-            if (param_has_anno and not isinstance(param_py_type, type) and
-                    not typing_inspect.is_generic_type(param_py_type)):
+            if (param_has_anno and not isinstance(param_py_type, type)
+               and not typing_inspect.is_generic_type(param_py_type)):
                 raise FunctionLoadError(
                     func_name,
                     f'binding {param.name} has invalid non-type annotation '
@@ -218,8 +218,8 @@ class Registry:
         return_pytype = None
         if return_binding_name is not None and 'return' in annotations:
             return_anno = annotations.get('return')
-            if (typing_inspect.is_generic_type(return_anno) and
-                    typing_inspect.get_origin(return_anno).__name__ == 'Out'):
+            if (typing_inspect.is_generic_type(return_anno)
+               and typing_inspect.get_origin(return_anno).__name__ == 'Out'):
                 raise FunctionLoadError(
                     func_name,
                     f'return annotation should not be azure.functions.Out')

azure_functions_worker/testutils.py

@@ -103,8 +103,8 @@ class AsyncTestCaseMeta(type(unittest.TestCase)):
 
     def __new__(mcls, name, bases, ns):
         for attrname, attr in ns.items():
-            if (attrname.startswith('test_') and
-                    inspect.iscoroutinefunction(attr)):
+            if (attrname.startswith('test_')
+               and inspect.iscoroutinefunction(attr)):
                 ns[attrname] = mcls._sync_wrap(attr)
 
         return super().__new__(mcls, name, bases, ns)
@@ -348,7 +348,7 @@ class _MockWebHost:
             name,
             input_data: typing.List[protos.ParameterBinding],
             metadata: typing.Optional[
-                typing.Mapping[str, protos.TypedData]]=None):
+                typing.Mapping[str, protos.TypedData]] = None):
 
         if metadata is None:
             metadata = {}

pack/Microsoft.Azure.Functions.V2.PythonWorker.nuspec

@@ -19,6 +19,6 @@
     <file src="..\3.7_LINUX_X64\**" target="tools\3.7\LINUX\X64" />
     <file src="..\3.7_OSX_X64\**" target="tools\3.7\OSX\X64" />
     <file src="..\python\prod\worker.config.json" target="tools" />
-    <file src="Microsoft.Azure.Functions.PythonWorker.targets" target="build" />
+    <file src="Microsoft.Azure.Functions.V2.PythonWorker.targets" target="build" />
   </files>
-</package>
+</package>

pack/Microsoft.Azure.Functions.V2.PythonWorker.targets

@@ -14,12 +14,12 @@
   </Target>
 
   <Target Name="CopyOnPublish" DependsOnTargets="Initialize" AfterTargets="Publish">
-        <Copy SourceFiles="@(SourceFiles)" 
+        <Copy SourceFiles="@(SourceFiles)"
             DestinationFiles="@(SourceFiles->'$(PublishDir)\workers\python\%(RecursiveDir)%(Filename)%(Extension)')" />
   </Target>
 
   <Target Name="CopyOnBuild" DependsOnTargets="Initialize" AfterTargets="Build">
-        <Copy SourceFiles="@(SourceFiles)" 
+        <Copy SourceFiles="@(SourceFiles)"
             DestinationFiles="@(SourceFiles->'$(OutDir)\workers\python\%(RecursiveDir)%(Filename)%(Extension)')" />
   </Target>
-</Project>
+</Project>

pack/Microsoft.Azure.Functions.V3.PythonWorker.nuspec

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
+  <metadata>
+    <id>Microsoft.Azure.Functions.PythonWorker</id>
+    <version>1.1.0</version>
+    <authors>Microsoft</authors>
+    <owners>Microsoft</owners>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <description>Microsoft Azure Functions Python Worker</description>
+    <copyright>© .NET Foundation. All rights reserved.</copyright>
+  </metadata>
+  <files>
+    <file src="..\3.6_WINDOWS_x64\**" target="tools\3.6\WINDOWS\X64" />
+    <!-- <file src="..\3.6_WINDOWS_X86\**" target="tools\3.6\WINDOWS\X86" /> -->
+    <file src="..\3.6_LINUX_X64\**" target="tools\3.6\LINUX\X64" />
+    <file src="..\3.6_OSX_X64\**" target="tools\3.6\OSX\X64" />
+    <file src="..\3.7_WINDOWS_X64\**" target="tools\3.7\WINDOWS\X64" />
+    <file src="..\3.7_WINDOWS_X86\**" target="tools\3.7\WINDOWS\X86" />
+    <file src="..\3.7_LINUX_X64\**" target="tools\3.7\LINUX\X64" />
+    <file src="..\3.7_OSX_X64\**" target="tools\3.7\OSX\X64" />
+    <file src="..\3.8_WINDOWS_X64\**" target="tools\3.8\WINDOWS\X64" />
+    <file src="..\3.8_WINDOWS_X86\**" target="tools\3.8\WINDOWS\X86" />
+    <file src="..\3.8_LINUX_X64\**" target="tools\3.8\LINUX\X64" />
+    <file src="..\3.8_OSX_X64\**" target="tools\3.8\OSX\X64" />
+    <file src="..\python\prod\worker.config.json" target="tools" />
+    <file src="Microsoft.Azure.Functions.V3.PythonWorker.targets" target="build" />
+  </files>
+</package>

pack/Microsoft.Azure.Functions.V3.PythonWorker.targets

@@ -0,0 +1,25 @@
+<Project>
+  <Target Name="Initialize" AfterTargets="Publish;Build">
+    <CreateItem Include="$(MSBuildThisFileDirectory)\..\">
+      <Output ItemName="MSBuildThisFileDirectoryParentDirectory" TaskParameter="Include"/>
+    </CreateItem>
+
+    <CreateProperty Value="%(MSBuildThisFileDirectoryParentDirectory.Fullpath)">
+        <Output PropertyName="NugetRoot" TaskParameter="Value"/>
+    </CreateProperty>
+
+    <ItemGroup>
+        <SourceFiles Include="$(NugetRoot)tools\**\*.*"/>
+    </ItemGroup>
+  </Target>
+
+  <Target Name="CopyOnPublish" DependsOnTargets="Initialize" AfterTargets="Publish">
+        <Copy SourceFiles="@(SourceFiles)"
+            DestinationFiles="@(SourceFiles->'$(PublishDir)\workers\python\%(RecursiveDir)%(Filename)%(Extension)')" />
+  </Target>
+
+  <Target Name="CopyOnBuild" DependsOnTargets="Initialize" AfterTargets="Build">
+        <Copy SourceFiles="@(SourceFiles)"
+            DestinationFiles="@(SourceFiles->'$(OutDir)\workers\python\%(RecursiveDir)%(Filename)%(Extension)')" />
+  </Target>
+</Project>

pack/scripts/win_deps.ps1

@@ -4,4 +4,4 @@ python -m pip install .
 
 $depsPath = Join-Path -Path $env:BUILD_SOURCESDIRECTORY -ChildPath "deps"
 
-python -m pip install . azure-functions --no-compile --target $depsPath.ToString()
+python -m pip install . azure-functions --no-compile --target $depsPath.ToString()

pack/templates/win_env_gen.yml

@@ -9,7 +9,7 @@ steps:
     architecture: ${{ parameters.architecture }}
     addToPath: true
 - task: PowerShell@2
-  inputs: 
+  inputs:
     filePath: 'pack\scripts\win_deps.ps1'
 - task: CopyFiles@2
   inputs:
@@ -22,7 +22,7 @@ steps:
     contents: |
       python\prod\worker.py
     targetFolder: '$(Build.ArtifactStagingDirectory)'
-    flattenFolders: true  
+    flattenFolders: true
 - task: CopyFiles@2
   inputs:
     sourceFolder: '$(Build.SourcesDirectory)\deps'
@@ -34,4 +34,4 @@ steps:
 - task: PublishBuildArtifacts@1
   inputs:
     pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: ${{ parameters.artifactName }}
+    artifactName: ${{ parameters.artifactName }}

python/prod/worker.py

@@ -16,8 +16,8 @@ AZURE_CONTAINER_NAME = "CONTAINER_NAME"
 
 
 def is_azure_environment():
-    return (AZURE_CONTAINER_NAME in os.environ or
-            AZURE_WEBSITE_INSTANCE_ID in os.environ)
+    return (AZURE_CONTAINER_NAME in os.environ
+            or AZURE_WEBSITE_INSTANCE_ID in os.environ)
 
 
 def determine_user_pkg_paths():

setup.py

@@ -239,18 +239,14 @@ setup(
     packages=['azure_functions_worker',
               'azure_functions_worker.protos',
               'azure_functions_worker.bindings'],
-    setup_requires=[
-        'grpcio~=1.20.1',
-        'grpcio-tools~=1.20.1',
-    ],
     install_requires=[
-        'grpcio~=1.20.1',
-        'grpcio-tools~=1.20.1',
+        'grpcio==1.26.0',
+        'grpcio-tools==1.26.0',
     ],
     extras_require={
         'dev': [
             'azure-functions==1.0.7',
-            'flake8~=3.5.0',
+            'flake8~=3.7.9',
             'mypy',
             'pytest',
             'requests==2.*',

tests/endtoend/test_cosmosdb_functions.py

@@ -37,7 +37,7 @@ class TestCosmosDBFunctions(testutils.WebHostTestCase):
                     response,
                     doc
                 )
-            except AssertionError as e:
+            except AssertionError:
                 if try_no == max_retries - 1:
                     raise
             else:
@@ -69,7 +69,7 @@ class TestCosmosDBFunctions(testutils.WebHostTestCase):
                     response,
                     doc
                 )
-            except AssertionError as e:
+            except AssertionError:
                 if try_no == max_retries - 1:
                     raise
             else:

tests/endtoend/test_eventgrid_functions.py

@@ -76,7 +76,7 @@ class TestEventGridFunctions(testutils.WebHostTestCase):
                         'event_type': data[0]['eventType'],
                     }
                 )
-            except AssertionError as e:
+            except AssertionError:
                 if try_no == max_retries - 1:
                     raise
             else:

tests/unittests/test_rpc_messages.py

@@ -14,25 +14,25 @@ class TestGRPC(testutils.AsyncTestCase):
             os.environ[key] = value
 
     async def _verify_environment_reloaded(self, test_env):
-            request = protos.FunctionEnvironmentReloadRequest(
-                environment_variables=test_env)
+        request = protos.FunctionEnvironmentReloadRequest(
+            environment_variables=test_env)
 
-            request_msg = protos.StreamingMessage(
-                request_id='0',
-                function_environment_reload_request=request)
+        request_msg = protos.StreamingMessage(
+            request_id='0',
+            function_environment_reload_request=request)
 
-            disp = testutils.create_dummy_dispatcher()
+        disp = testutils.create_dummy_dispatcher()
 
-            try:
-                r = await disp._handle__function_environment_reload_request(
-                    request_msg)
+        try:
+            r = await disp._handle__function_environment_reload_request(
+                request_msg)
 
-                environ_dict = os.environ.copy()
-                self.assertDictEqual(environ_dict, test_env)
-                status = r.function_environment_reload_response.result.status
-                self.assertEqual(status, protos.StatusResult.Success)
-            finally:
-                self._reset_environ()
+            environ_dict = os.environ.copy()
+            self.assertDictEqual(environ_dict, test_env)
+            status = r.function_environment_reload_response.result.status
+            self.assertEqual(status, protos.StatusResult.Success)
+        finally:
+            self._reset_environ()
 
     async def test_multiple_env_vars_load(self):
         test_env = {'TEST_KEY': 'foo', 'HELLO': 'world'}