feat: Update to support py312 (#1533)

* Update to support py312 * Fixed new relic unit tests * Adding 312 to unit tests * 312 typo * py312 flake8 fix * Removed 312 for consumption tests * Downgraded protobuf * Updated protobuf and grpc versions * Downgrading protobuf to < 5 * Reverted new relic unit test * Updated nuspec file * Fixed pipeline for sql e2e tests --------- Co-authored-by: hallvictoria <victoriahall@microsoft.com> Co-authored-by: hallvictoria <59299039+hallvictoria@users.noreply.github.com>
2024-07-22 17:03:55 -05:00 · 2024-07-22 17:03:55 -05:00 · bd14bc39a8
--- a/eng/templates/jobs/ci-unit-tests.yml
+++ b/eng/templates/jobs/ci-unit-tests.yml
@ -14,6 +14,8 @@ jobs:
          PYTHON_VERSION: '3.10'
        Python311:
          PYTHON_VERSION: '3.11'
+        Python312:
+          PYTHON_VERSION: '3.12'
    
    steps:
      - task: UsePythonVersion@0
--- a/eng/templates/official/jobs/build-artifacts.yml
+++ b/eng/templates/official/jobs/build-artifacts.yml
@ -21,6 +21,9 @@ jobs:
      Python311V4:
        pythonVersion: '3.11'
        workerPath: 'python/prodV4/worker.py'
+      Python312V4:
+        pythonVersion: '3.12'
+        workerPath: 'python/prodV4/worker.py'
  templateContext:
    outputParentDirectory: $(Build.ArtifactStagingDirectory)
    outputs:
--- a/eng/templates/official/jobs/ci-docker-consumption-tests.yml
+++ b/eng/templates/official/jobs/ci-docker-consumption-tests.yml
@ -45,6 +45,15 @@ jobs:
          SQL_CONNECTION: $(LinuxSqlConnectionString311)
          EVENTGRID_URI: $(LinuxEventGridTopicUriString311)
          EVENTGRID_CONNECTION: $(LinuxEventGridConnectionKeyString311)
+        Python312:
+          PYTHON_VERSION: '3.12'
+          STORAGE_CONNECTION: $(LinuxStorageConnectionString312)
+          COSMOSDB_CONNECTION: $(LinuxCosmosDBConnectionString312)
+          EVENTHUB_CONNECTION: $(LinuxEventHubConnectionString312)
+          SERVICEBUS_CONNECTION: $(LinuxServiceBusConnectionString312)
+          SQL_CONNECTION: $(LinuxSqlConnectionString312)
+          EVENTGRID_URI: $(LinuxEventGridTopicUriString312)
+          EVENTGRID_CONNECTION: $(LinuxEventGridConnectionKeyString312)
    
    steps:
      - task: UsePythonVersion@0
--- a/eng/templates/official/jobs/ci-docker-dedicated-tests.yml
+++ b/eng/templates/official/jobs/ci-docker-dedicated-tests.yml
@ -45,6 +45,15 @@ jobs:
          SQL_CONNECTION: $(LinuxSqlConnectionString311)
          EVENTGRID_URI: $(LinuxEventGridTopicUriString311)
          EVENTGRID_CONNECTION: $(LinuxEventGridConnectionKeyString311)
+        Python312:
+          PYTHON_VERSION: '3.12'
+          STORAGE_CONNECTION: $(LinuxStorageConnectionString312)
+          COSMOSDB_CONNECTION: $(LinuxCosmosDBConnectionString312)
+          EVENTHUB_CONNECTION: $(LinuxEventHubConnectionString312)
+          SERVICEBUS_CONNECTION: $(LinuxServiceBusConnectionString312)
+          SQL_CONNECTION: $(LinuxSqlConnectionString312)
+          EVENTGRID_URI: $(LinuxEventGridTopicUriString312)
+          EVENTGRID_CONNECTION: $(LinuxEventGridConnectionKeyString312)
    
    steps:
      - task: UsePythonVersion@0
--- a/eng/templates/official/jobs/ci-e2e-tests.yml
+++ b/eng/templates/official/jobs/ci-e2e-tests.yml
@ -54,6 +54,15 @@ jobs:
          SQL_CONNECTION: $(LinuxSqlConnectionString311)
          EVENTGRID_URI: $(LinuxEventGridTopicUriString311)
          EVENTGRID_CONNECTION: $(LinuxEventGridConnectionKeyString311)
+        Python312:
+          PYTHON_VERSION: '3.12'
+          STORAGE_CONNECTION: $(LinuxStorageConnectionString312)
+          COSMOSDB_CONNECTION: $(LinuxCosmosDBConnectionString312)
+          EVENTHUB_CONNECTION: $(LinuxEventHubConnectionString312)
+          SERVICEBUS_CONNECTION: $(LinuxServiceBusConnectionString312)
+          SQL_CONNECTION: $(LinuxSqlConnectionString312)
+          EVENTGRID_URI: $(LinuxEventGridTopicUriString312)
+          EVENTGRID_CONNECTION: $(LinuxEventGridConnectionKeyString312)
    steps:
      - task: UsePythonVersion@0
        inputs:
--- a/eng/templates/official/jobs/ci-lc-tests.yml
+++ b/eng/templates/official/jobs/ci-lc-tests.yml
@ -11,20 +11,15 @@ jobs:
      matrix:
        Python37:
          PYTHON_VERSION: '3.7'
-          STORAGE_CONNECTION: $(LinuxStorageConnectionString37)
        Python38:
          PYTHON_VERSION: '3.8'
-          STORAGE_CONNECTION: $(LinuxStorageConnectionString38)
        Python39:
          PYTHON_VERSION: '3.9'
-          STORAGE_CONNECTION: $(LinuxStorageConnectionString39)
        Python310:
          PYTHON_VERSION: '3.10'
-          STORAGE_CONNECTION: $(LinuxStorageConnectionString310)
        Python311:
          PYTHON_VERSION: '3.11'
-          STORAGE_CONNECTION: $(LinuxStorageConnectionString311)
-    
+
    steps:
      - task: UsePythonVersion@0
        inputs:
@ -39,6 +34,6 @@ jobs:
      - bash: |
             python -m pytest -n auto --dist loadfile -vv  --reruns 4 --instafail tests/consumption_tests
        env:
-          AzureWebJobsStorage: $(STORAGE_CONNECTION)
+          AzureWebJobsStorage: $(LinuxStorageConnectionString312)
          _DUMMY_CONT_KEY: $(_DUMMY_CONT_KEY)
        displayName: "Running $(PYTHON_VERSION) Linux Consumption tests"
--- a/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec
+++ b/pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec
@ -33,6 +33,11 @@
    <file src="..\3.11_LINUX_X64\**" target="tools\3.11\LINUX\X64" />
    <file src="..\3.11_OSX_X64\**" target="tools\3.11\OSX\X64" />
    <file src="..\3.11_OSX_X64\**" target="tools\3.11\OSX\Arm64" />
+    <file src="..\3.12_WINDOWS_X64\**" target="tools\3.12\WINDOWS\X64" />
+    <file src="..\3.12_WINDOWS_X86\**" target="tools\3.12\WINDOWS\X86" />
+    <file src="..\3.12_LINUX_X64\**" target="tools\3.12\LINUX\X64" />
+    <file src="..\3.12_OSX_X64\**" target="tools\3.12\OSX\X64" />
+    <file src="..\3.12_OSX_X64\**" target="tools\3.12\OSX\Arm64" />
    <file src="..\python\prodV4\worker.config.json" target="tools" />
    <file src="Microsoft.Azure.Functions.PythonWorker.targets" target="build" />
    <file src="..\_manifest\spdx_2.2\manifest.spdx.json" target="SBOM\manifest.spdx.json" />
--- a/pyproject.toml
+++ b/pyproject.toml
@ -29,11 +29,11 @@ dependencies = [
     "azure-functions==1.20.0",
     "python-dateutil ~=2.9.0",
     "protobuf~=3.19.3; python_version == '3.7'",
-     "protobuf~=4.22.0; python_version >= '3.8'",
+     "protobuf~=4.25.3; python_version >= '3.8'",
     "grpcio-tools~=1.43.0; python_version == '3.7'",
-     "grpcio-tools~=1.54.2; python_version >= '3.8'",
+     "grpcio-tools~=1.59.0; python_version >= '3.8'",
     "grpcio~=1.43.0; python_version == '3.7'",
-     "grpcio~=1.54.2; python_version >= '3.8'",
+     "grpcio~=1.59.0; python_version >= '3.8'",
     "azurefunctions-extensions-base; python_version >= '3.8'"
 ]

@ -49,7 +49,8 @@ dev = [
    "fastapi~=0.103.2",
    "pydantic",
    "pycryptodome==3.*",
-    "flake8==5.*",
+    "flake8==5.*; python_version == '3.7'",
+    "flake8==6.*; python_version >= '3.8'",
    "mypy",
    "pytest~=7.4.4",
    "requests==2.*",
--- a/python/prodV4/worker.config.json
+++ b/python/prodV4/worker.config.json
@ -1,9 +1,9 @@
 {
    "description":{
        "language":"python",
-        "defaultRuntimeVersion":"3.10",
+        "defaultRuntimeVersion":"3.11",
        "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"],
-        "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10", "3.11"],
+        "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"],
        "supportedArchitectures":["X64", "X86", "Arm64"],
        "extensions":[".py"],
        "defaultExecutablePath":"python",
--- a/python/prodV4/worker.py
+++ b/python/prodV4/worker.py
@ -35,13 +35,10 @@ def determine_user_pkg_paths():
    """
    minor_version = sys.version_info[1]

-    usr_packages_path = []
-
-    if minor_version in (7, 8, 9, 10, 11):
-        usr_packages_path.append(os.path.join(PKGS_PATH, PKGS))
-    else:
+    if not (7 <= minor_version <= 12):
        raise RuntimeError(f'Unsupported Python version: 3.{minor_version}')

+    usr_packages_path = [os.path.join(PKGS_PATH, PKGS)]
    return usr_packages_path


--- a/tests/test_setup.py
+++ b/tests/test_setup.py
@ -31,7 +31,7 @@ from distutils import dir_util

 from invoke import task

-from tests.utils.constants import EXTENSIONS_CSPROJ_TEMPLATE, NUGET_CONFIG
+from utils.constants import EXTENSIONS_CSPROJ_TEMPLATE, NUGET_CONFIG

 ROOT_DIR = pathlib.Path(__file__).parent.parent
 BUILD_DIR = ROOT_DIR / 'build'
--- a/tests/unittests/test_utilities.py
+++ b/tests/unittests/test_utilities.py
@ -320,13 +320,15 @@ class TestUtilities(unittest.TestCase):
        is_python_version_39 = common.is_python_version('3.9')
        is_python_version_310 = common.is_python_version('3.10')
        is_python_version_311 = common.is_python_version('3.11')
+        is_python_version_312 = common.is_python_version('3.12')

        self.assertTrue(any([
            is_python_version_37,
            is_python_version_38,
            is_python_version_39,
            is_python_version_310,
-            is_python_version_311
+            is_python_version_311,
+            is_python_version_312
        ]))

    def test_get_sdk_from_sys_path(self):