diff --git a/.ci/e2e_integration_test/start-e2e.ps1 b/.ci/e2e_integration_test/start-e2e.ps1
index 61e7f0bf..653ed0bc 100644
--- a/.ci/e2e_integration_test/start-e2e.ps1
+++ b/.ci/e2e_integration_test/start-e2e.ps1
@@ -89,8 +89,9 @@ Write-Host "Preparing E2E integration tests..." -ForegroundColor Green
Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
python -m pip install -U pip
python -m pip install -U -e .[dev]
-python setup.py build
-python setup.py extension
+cd tests
+python -m invoke -c test_setup build-protos
+python -m invoke -c test_setup extensions
Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 42a31660..03ff4780 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -46,7 +46,7 @@
// "forwardPorts": [],
// Use 'postCreateCommand' to run commands after the container is created.
- "postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && sudo python setup.py webhost",
+ "postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && cd tests && sudo python -m invoke -c test_setup webhost",
// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode",
diff --git a/dev-setup.py b/dev-setup.py
deleted file mode 100644
index 86de8d2b..00000000
--- a/dev-setup.py
+++ /dev/null
@@ -1,505 +0,0 @@
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License.
-
-import distutils.cmd
-import glob
-import json
-import os
-import pathlib
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-import urllib.request
-import zipfile
-from distutils import dir_util
-from distutils.command import build
-from distutils.dist import Distribution
-
-from setuptools import setup
-from setuptools.command import develop
-
-from azure_functions_worker.version import VERSION
-from tests.utils.constants import EXTENSIONS_CSPROJ_TEMPLATE
-
-# The GitHub repository of the Azure Functions Host
-WEBHOST_GITHUB_API = "https://api.github.com/repos/Azure/azure-functions-host"
-WEBHOST_TAG_PREFIX = "v4."
-WEBHOST_GIT_REPO = "https://github.com/Azure/azure-functions-host/archive"
-NUGET_CONFIG = """\
-
-
-
-
-
-
-
-
-
-
-"""
-
-CLASSIFIERS = [
- "Development Status :: 5 - Production/Stable",
- "Programming Language :: Python",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3.9",
- "Operating System :: Microsoft :: Windows",
- "Operating System :: POSIX",
- "Operating System :: MacOS :: MacOS X",
- "Environment :: Web Environment",
- "License :: OSI Approved :: MIT License",
- "Intended Audience :: Developers",
-]
-
-PACKAGES = [
- "azure_functions_worker",
- "azure_functions_worker.protos",
- "azure_functions_worker.protos.identity",
- "azure_functions_worker.protos.shared",
- "azure_functions_worker.bindings",
- "azure_functions_worker.bindings.shared_memory_data_transfer",
- "azure_functions_worker.utils",
- "azure_functions_worker._thirdparty",
-]
-
-INSTALL_REQUIRES = ["azure-functions==1.20.0", "python-dateutil~=2.8.2"]
-
-if sys.version_info[:2] == (3, 7):
- INSTALL_REQUIRES.extend(
- ("protobuf~=3.19.3", "grpcio-tools~=1.43.0", "grpcio~=1.43.0")
- )
-else:
- INSTALL_REQUIRES.extend(
- ("protobuf~=4.22.0", "grpcio-tools~=1.54.2", "grpcio~=1.54.2",
- "azurefunctions-extensions-base")
- )
-
-EXTRA_REQUIRES = {
- "dev": [
- "azure-eventhub", # Used for EventHub E2E tests
- "azure-functions-durable", # Used for Durable E2E tests
- "flask",
- "fastapi~=0.85.0", # Used for ASGIMiddleware test
- "pydantic",
- "pycryptodome~=3.10.1",
- "flake8~=4.0.1",
- "mypy",
- "pytest~=7.4.4",
- "requests==2.*",
- "coverage",
- "pytest-sugar",
- "pytest-cov",
- "pytest-xdist",
- "pytest-randomly",
- "pytest-instafail",
- "pytest-rerunfailures",
- "ptvsd",
- "python-dotenv",
- "plotly",
- "scikit-learn",
- "opencv-python",
- "pandas",
- "numpy",
- "pre-commit"
- ],
- "test-http-v2": [
- "azurefunctions-extensions-http-fastapi",
- "ujson",
- "orjson"
- ],
- "test-deferred-bindings": [
- "azurefunctions-extensions-bindings-blob"
- ]
-}
-
-
-class BuildGRPC:
- """Generate gRPC bindings."""
-
- def _gen_grpc(self):
- root = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
-
- proto_root_dir = root / "azure_functions_worker" / "protos"
- proto_src_dir = proto_root_dir / "_src" / "src" / "proto"
- build_dir = root / "build"
- staging_root_dir = build_dir / "protos"
- staging_dir = staging_root_dir / "azure_functions_worker" / "protos"
- built_protos_dir = build_dir / "built_protos"
-
- if os.path.exists(build_dir):
- shutil.rmtree(build_dir)
-
- shutil.copytree(proto_src_dir, staging_dir)
-
- os.makedirs(built_protos_dir)
-
- protos = [
- os.sep.join(("shared", "NullableTypes.proto")),
- os.sep.join(("identity", "ClaimsIdentityRpc.proto")),
- "FunctionRpc.proto",
- ]
-
- for proto in protos:
- subprocess.run(
- [
- sys.executable,
- "-m",
- "grpc_tools.protoc",
- "-I",
- os.sep.join(("azure_functions_worker", "protos")),
- "--python_out",
- str(built_protos_dir),
- "--grpc_python_out",
- str(built_protos_dir),
- os.sep.join(("azure_functions_worker", "protos", proto)),
- ],
- check=True,
- stdout=sys.stdout,
- stderr=sys.stderr,
- cwd=staging_root_dir,
- )
-
- compiled_files = glob.glob(
- str(built_protos_dir / "**" / "*.py"), recursive=True
- )
-
- if not compiled_files:
- print("grpc_tools.protoc produced no Python files", file=sys.stderr)
- sys.exit(1)
-
- # Needed to support absolute imports in files. See
- # https://github.com/protocolbuffers/protobuf/issues/1491
- self.make_absolute_imports(compiled_files)
-
- dir_util.copy_tree(str(built_protos_dir), str(proto_root_dir))
-
- @staticmethod
- def make_absolute_imports(compiled_files):
- for compiled in compiled_files:
- with open(compiled, "r+") as f:
- content = f.read()
- f.seek(0)
- # Convert lines of the form:
- # import xxx_pb2 as xxx__pb2 to
- # from azure_functions_worker.protos import xxx_pb2 as..
- p1 = re.sub(
- r"\nimport (.*?_pb2)",
- r"\nfrom azure_functions_worker.protos import \g<1>",
- content,
- )
- # Convert lines of the form:
- # from identity import xxx_pb2 as.. to
- # from azure_functions_worker.protos.identity import xxx_pb2..
- p2 = re.sub(
- r"from ([a-z]*) (import.*_pb2)",
- r"from azure_functions_worker.protos.\g<1> \g<2>",
- p1,
- )
- f.write(p2)
- f.truncate()
-
-
-class BuildProtos(build.build, BuildGRPC):
- def run(self, *args, **kwargs):
- self._gen_grpc()
- super().run()
-
-
-class Development(develop.develop, BuildGRPC):
- def run(self, *args, **kwargs):
- self._gen_grpc()
- super().run()
-
-
-class Extension(distutils.cmd.Command):
- description = "Resolve WebJobs Extensions from AZURE_EXTENSIONS and NUGET_CONFIG."
- user_options = [
- (
- "extensions-dir",
- None,
- "A path to the directory where extension should be installed",
- )
- ]
-
- def __init__(self, dist: Distribution):
- super().__init__(dist)
- self.extensions_dir = None
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- if self.extensions_dir is None:
- self.extensions_dir = pathlib.Path(__file__).parent / "build" / "extensions"
-
- def _install_extensions(self):
- if not self.extensions_dir.exists():
- os.makedirs(self.extensions_dir, exist_ok=True)
-
- if not (self.extensions_dir / "host.json").exists():
- with open(self.extensions_dir / "host.json", "w") as f:
- print("{}", file=f)
-
- if not (self.extensions_dir / "extensions.csproj").exists():
- with open(self.extensions_dir / "extensions.csproj", "w") as f:
- print(EXTENSIONS_CSPROJ_TEMPLATE, file=f)
-
- with open(self.extensions_dir / "NuGet.config", "w") as f:
- print(NUGET_CONFIG, file=f)
-
- env = os.environ.copy()
- env["TERM"] = "xterm" # ncurses 6.1 workaround
- try:
- subprocess.run(
- args=["dotnet", "build", "-o", "."],
- check=True,
- cwd=str(self.extensions_dir),
- stdout=sys.stdout,
- stderr=sys.stderr,
- env=env,
- )
- except Exception: # NoQA
- print(
- ".NET Core SDK is required to build the extensions. "
- "Please visit https://aka.ms/dotnet-download"
- )
- sys.exit(1)
-
- def run(self):
- self._install_extensions()
-
-
-class Webhost(distutils.cmd.Command):
- description = "Download and setup Azure Functions Web Host."
- user_options = [
- (
- "webhost-version=",
- None,
- "A Functions Host version to be downloaded (e.g. 3.0.15278).",
- ),
- (
- "webhost-dir=",
- None,
- "A path to the directory where Azure Web Host will be installed.",
- ),
- (
- "branch-name=",
- None,
- "A branch from where azure-functions-host will be installed "
- "(e.g. branch-name=dev, branch-name= abc/branchname)",
- ),
- ]
-
- def __init__(self, dist: Distribution):
- super().__init__(dist)
- self.webhost_dir = None
- self.webhost_version = None
- self.branch_name = None
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- if self.webhost_version is None:
- self.webhost_version = self._get_webhost_version()
-
- if self.webhost_dir is None:
- self.webhost_dir = pathlib.Path(__file__).parent / "build" / "webhost"
-
- @staticmethod
- def _get_webhost_version() -> str:
- # Return the latest matched version (e.g. 3.0.15278)
- github_api_url = f"{WEBHOST_GITHUB_API}/tags?page=1&per_page=10"
- print(f"Checking latest webhost version from {github_api_url}")
- github_response = urllib.request.urlopen(github_api_url)
- tags = json.loads(github_response.read())
-
- # As tags are placed in time desending order, the latest v3
- # tag should be the first occurance starts with 'v3.' string
- latest = [gt for gt in tags if gt["name"].startswith(WEBHOST_TAG_PREFIX)]
- return latest[0]["name"].replace("v", "")
-
- @staticmethod
- def _download_webhost_zip(version: str, branch: str) -> str:
- # Return the path of the downloaded host
- temporary_file = tempfile.NamedTemporaryFile()
-
- if branch is not None:
- zip_url = f"{WEBHOST_GIT_REPO}/refs/heads/{branch}.zip"
- else:
- zip_url = f"{WEBHOST_GIT_REPO}/v{version}.zip"
-
- print(f"Downloading Functions Host from {zip_url}")
-
- with temporary_file as zipf:
- zipf.close()
- try:
- urllib.request.urlretrieve(zip_url, zipf.name)
- except Exception as e:
- print(
- "Failed to download Functions Host source code from"
- f" {zip_url}: {e!r}",
- file=sys.stderr,
- )
- sys.exit(1)
-
- print(f"Functions Host is downloaded into {temporary_file.name}")
- return temporary_file.name
-
- @staticmethod
- def _create_webhost_folder(dest_folder: pathlib.Path):
- if dest_folder.exists():
- shutil.rmtree(dest_folder)
- os.makedirs(dest_folder, exist_ok=True)
- print(f"Functions Host folder is created in {dest_folder}")
-
- @staticmethod
- def _extract_webhost_zip(version: str, src_zip: str, dest: str):
- print(f"Extracting Functions Host from {src_zip}")
-
- with zipfile.ZipFile(src_zip) as archive:
- # We cannot simply use extractall(), as the archive
- # contains Windows-style path names, which are not
- # automatically converted into Unix-style paths, so
- # extractall() will produce a flat directory with
- # backslashes in file names.
-
- for archive_name in archive.namelist():
- prefix = f"azure-functions-host-{version}/"
- if archive_name.startswith(prefix):
- sanitized_name = archive_name.replace("\\", os.sep).replace(
- prefix, ""
- )
- dest_filename = dest / sanitized_name
- zipinfo = archive.getinfo(archive_name)
-
- try:
- if not dest_filename.parent.exists():
- os.makedirs(dest_filename.parent, exist_ok=True)
-
- if zipinfo.is_dir():
- os.makedirs(dest_filename, exist_ok=True)
- else:
- with archive.open(archive_name) as src, open(
- dest_filename, "wb"
- ) as dst:
- dst.write(src.read())
- except Exception as e:
- print(
- f"Failed to extract file {archive_name}" f": {e!r}",
- file=sys.stderr,
- )
- sys.exit(1)
-
- print(f"Functions Host is extracted into {dest}")
-
- @staticmethod
- def _chmod_protobuf_generation_script(webhost_dir: pathlib.Path):
- # This script is needed to set to executable in order to build the
- # WebJobs.Script.Grpc project in Linux and MacOS
- script_path = webhost_dir / "src" / "WebJobs.Script.Grpc" / "generate_protos.sh"
- if sys.platform != "win32" and os.path.exists(script_path):
- print("Change generate_protos.sh script permission")
- os.chmod(script_path, 0o555)
-
- @staticmethod
- def _compile_webhost(webhost_dir: pathlib.Path):
- print(f"Compiling Functions Host from {webhost_dir}")
-
- try:
- subprocess.run(
- args=["dotnet", "build", "WebJobs.Script.sln", "-o", "bin",
- "/p:TreatWarningsAsErrors=false"],
- check=True,
- cwd=str(webhost_dir),
- stdout=sys.stdout,
- stderr=sys.stderr,
- )
- except Exception: # NoQA
- print(
- f"Failed to compile webhost in {webhost_dir}. "
- ".NET Core SDK is required to build the solution. "
- "Please visit https://aka.ms/dotnet-download",
- file=sys.stderr,
- )
- sys.exit(1)
-
- print("Functions Host is compiled successfully")
-
- def run(self):
- # Prepare webhost
- zip_path = self._download_webhost_zip(self.webhost_version, self.branch_name)
- self._create_webhost_folder(self.webhost_dir)
- version = self.branch_name or self.webhost_version
- self._extract_webhost_zip(
- version=version.replace("/", "-"), src_zip=zip_path, dest=self.webhost_dir
- )
- self._chmod_protobuf_generation_script(self.webhost_dir)
- self._compile_webhost(self.webhost_dir)
-
-
-class Clean(distutils.cmd.Command):
- description = "Clean up build generated files"
- user_options = []
-
- def __init__(self, dist: Distribution):
- super().__init__(dist)
- self.dir_list_to_delete = ["build"]
-
- def initialize_options(self) -> None:
- pass
-
- def finalize_options(self) -> None:
- pass
-
- def run(self) -> None:
- for dir_to_delete in self.dir_list_to_delete:
- dir_delete = pathlib.Path(dir_to_delete)
- if dir_delete.exists():
- try:
- print(f"Deleting directory: {dir_to_delete}")
- shutil.rmtree(dir_delete)
- except OSError as ex:
- print(
- f"Error deleting directory: {dir_to_delete}. "
- f"Exception: {ex}"
- )
-
-#
-# COMMAND_CLASS = {
-# "develop": Development,
-# "build": BuildProtos,
-# "webhost": Webhost,
-# "webhost --branch-name={branch-name}": Webhost,
-# "extension": Extension,
-# "clean": Clean,
-# }
-
-# setup(
-# name="azure-functions-worker",
-# version=VERSION,
-# description="Python Language Worker for Azure Functions Host",
-# author="Azure Functions team at Microsoft Corp.",
-# author_email="azurefunctions@microsoft.com",
-# keywords="azure functions azurefunctions python serverless",
-# url="https://github.com/Azure/azure-functions-python-worker",
-# long_description=open("README.md").read(),
-# long_description_content_type="text/markdown",
-# classifiers=CLASSIFIERS,
-# license="MIT",
-# packages=PACKAGES,
-# install_requires=INSTALL_REQUIRES,
-# extras_require=EXTRA_REQUIRES,
-# include_package_data=True,
-# cmdclass=COMMAND_CLASS,
-# test_suite="tests",
-# )
diff --git a/eng/templates/jobs/build.yml b/eng/templates/jobs/build.yml
index 1c96122f..c556c8d5 100644
--- a/eng/templates/jobs/build.yml
+++ b/eng/templates/jobs/build.yml
@@ -17,6 +17,6 @@ jobs:
- bash: |
python -m venv .env
.env\Scripts\Activate.ps1
- python -m pip install --upgrade pip==23.0
+ python -m pip install --upgrade pip
python -m pip install .
displayName: 'Build python worker'
\ No newline at end of file
diff --git a/eng/templates/jobs/ci-unit-tests.yml b/eng/templates/jobs/ci-unit-tests.yml
index 314503d3..788c399e 100644
--- a/eng/templates/jobs/ci-unit-tests.yml
+++ b/eng/templates/jobs/ci-unit-tests.yml
@@ -32,9 +32,10 @@ jobs:
python -m pip install --pre -U -e .[test-http-v2]
fi
- python setup.py build
- python setup.py webhost --branch-name=dev
- python setup.py extension
+ cd tests
+ python -m invoke -c test_setup build-protos
+ python -m invoke -c test_setup webhost --branch-name=dev
+ python -m invoke -c test_setup extensions
displayName: "Install dependencies"
- bash: |
python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
diff --git a/eng/templates/official/jobs/ci-custom-image-tests.yml b/eng/templates/official/jobs/ci-custom-image-tests.yml
index 71897506..2a280fa3 100644
--- a/eng/templates/official/jobs/ci-custom-image-tests.yml
+++ b/eng/templates/official/jobs/ci-custom-image-tests.yml
@@ -19,7 +19,8 @@ jobs:
if [[ $(PYTHON_VERSION) != "3.7" && $(PYTHON_VERSION) != "3.8" ]]; then
python -m pip install --pre -U -e .[test-deferred-bindings]
fi
- python setup.py build
+ cd tests
+ python -m invoke -c test_setup build-protos
displayName: 'Install dependencies'
- bash: |
python -m pytest --reruns 4 -vv --instafail tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests
diff --git a/eng/templates/official/jobs/ci-docker-consumption-tests.yml b/eng/templates/official/jobs/ci-docker-consumption-tests.yml
index c769bfd4..56d2e8f6 100644
--- a/eng/templates/official/jobs/ci-docker-consumption-tests.yml
+++ b/eng/templates/official/jobs/ci-docker-consumption-tests.yml
@@ -56,7 +56,8 @@ jobs:
if [[ $(PYTHON_VERSION) != "3.8" ]]; then
python -m pip install --pre -U -e .[test-deferred-bindings]
fi
- python setup.py build
+ cd tests
+ python -m invoke -c test_setup build-protos
displayName: 'Install dependencies'
- bash: |
python -m pytest --reruns 4 -vv --instafail tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests
diff --git a/eng/templates/official/jobs/ci-docker-dedicated-tests.yml b/eng/templates/official/jobs/ci-docker-dedicated-tests.yml
index f1374c8d..82f44ef8 100644
--- a/eng/templates/official/jobs/ci-docker-dedicated-tests.yml
+++ b/eng/templates/official/jobs/ci-docker-dedicated-tests.yml
@@ -56,7 +56,8 @@ jobs:
if [[ $(PYTHON_VERSION) != "3.8" ]]; then
python -m pip install --pre -U -e .[test-deferred-bindings]
fi
- python setup.py build
+ cd tests
+ python -m invoke -c test_setup build-protos
displayName: 'Install dependencies'
- bash: |
python -m pytest --reruns 4 -vv --instafail tests/endtoend tests/extension_tests/deferred_bindings_tests tests/extension_tests/http_v2_tests
diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml
index efa46257..0362d628 100644
--- a/eng/templates/official/jobs/ci-e2e-tests.yml
+++ b/eng/templates/official/jobs/ci-e2e-tests.yml
@@ -74,9 +74,10 @@ jobs:
python -m pip install --pre -U -e .[test-deferred-bindings]
fi
- python setup.py build
- python setup.py webhost --branch-name=dev
- python setup.py extension
+ cd tests
+ python -m invoke -c test_setup build-protos
+ python -m invoke -c test_setup webhost --branch-name=dev
+ python -m invoke -c test_setup extensions
displayName: 'Install dependencies and the worker'
condition: and(eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false))
- task: DownloadPipelineArtifact@2
@@ -101,9 +102,10 @@ jobs:
python -m pip install --pre -U -e .[test-deferred-bindings]
fi
- python setup.py build
- python setup.py webhost --branch-name=dev
- python setup.py extension
+ cd tests
+ python -m invoke -c test_setup build-protos
+ python -m invoke -c test_setup webhost --branch-name=dev
+ python -m invoke -c test_setup extensions
displayName: 'Install test python sdk, dependencies and the worker'
condition: eq(variables['USETESTPYTHONSDK'], true)
- task: DownloadPipelineArtifact@2
@@ -130,9 +132,10 @@ jobs:
python -m pip install -U -e .[dev]
- python setup.py build
- python setup.py webhost --branch-name=dev
- python setup.py extension
+ cd tests
+ python -m invoke -c test_setup build-protos
+ python -m invoke -c test_setup webhost --branch-name=dev
+ python -m invoke -c test_setup extensions
displayName: 'Install test python extension, dependencies and the worker'
condition: eq(variables['USETESTPYTHONEXTENSIONS'], true)
- bash: |
diff --git a/eng/templates/official/jobs/ci-lc-tests.yml b/eng/templates/official/jobs/ci-lc-tests.yml
index c3642acc..a4322cad 100644
--- a/eng/templates/official/jobs/ci-lc-tests.yml
+++ b/eng/templates/official/jobs/ci-lc-tests.yml
@@ -38,7 +38,8 @@ jobs:
python -m pip install --pre -U -e .[test-http-v2]
fi
- python setup.py build
+ cd tests
+ python -m invoke -c test_setup build-protos
displayName: 'Install dependencies and the worker'
- bash: |
python -m pytest -n auto --dist loadfile -vv --reruns 4 --instafail tests/consumption_tests
diff --git a/pack/scripts/mac_arm64_deps.sh b/pack/scripts/mac_arm64_deps.sh
index 13a2315d..33d798d6 100644
--- a/pack/scripts/mac_arm64_deps.sh
+++ b/pack/scripts/mac_arm64_deps.sh
@@ -2,7 +2,7 @@
python -m venv .env
source .env/bin/activate
-python -m pip install --upgrade pip==23.0
+python -m pip install --upgrade pip
python -m pip install .
diff --git a/pack/scripts/nix_deps.sh b/pack/scripts/nix_deps.sh
index 13a2315d..33d798d6 100644
--- a/pack/scripts/nix_deps.sh
+++ b/pack/scripts/nix_deps.sh
@@ -2,7 +2,7 @@
python -m venv .env
source .env/bin/activate
-python -m pip install --upgrade pip==23.0
+python -m pip install --upgrade pip
python -m pip install .
diff --git a/pack/scripts/win_deps.ps1 b/pack/scripts/win_deps.ps1
index 1ad0e42c..d674915d 100644
--- a/pack/scripts/win_deps.ps1
+++ b/pack/scripts/win_deps.ps1
@@ -1,6 +1,6 @@
python -m venv .env
.env\Scripts\Activate.ps1
-python -m pip install --upgrade pip==23.0
+python -m pip install --upgrade pip
python -m pip install .
diff --git a/pyproject.toml b/pyproject.toml
index 3b899235..642aba66 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,7 +67,8 @@ dev = [
"opencv-python",
"pandas",
"numpy",
- "pre-commit"
+ "pre-commit",
+ "invoke"
]
test-http-v2 = [
"azurefunctions-extensions-http-fastapi",
diff --git a/setup.cfg b/setup.cfg
index b323ebf1..6f5a7fb9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,4 +17,4 @@ ignore_missing_imports = True
ignore_errors = True
[mypy-azure_functions_worker._thirdparty.typing_inspect]
-ignore_errors = True
\ No newline at end of file
+ignore_errors = True
diff --git a/tests/test_setup.py b/tests/test_setup.py
new file mode 100644
index 00000000..7dd68612
--- /dev/null
+++ b/tests/test_setup.py
@@ -0,0 +1,304 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+"""
+Usage:
+This file defines tasks for building Protos, webhost and extensions
+
+To use these tasks, you can run the following commands:
+
+1. Build protos:
+ invoke -c test_setup build-protos
+
+2. Set up the Azure Functions Web Host:
+ invoke -c test_setup webhost
+
+3. Install WebJobs extensions:
+ invoke -c test_setup extensions
+"""
+
+import glob
+import json
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib.request
+import zipfile
+from distutils import dir_util
+
+from invoke import task
+
+from tests.utils.constants import EXTENSIONS_CSPROJ_TEMPLATE, NUGET_CONFIG
+
+ROOT_DIR = pathlib.Path(__file__).parent.parent
+BUILD_DIR = ROOT_DIR / 'build'
+WEBHOST_GITHUB_API = "https://api.github.com/repos/Azure/azure-functions-host"
+WEBHOST_GIT_REPO = "https://github.com/Azure/azure-functions-host/archive"
+WEBHOST_TAG_PREFIX = "v4."
+
+
+def get_webhost_version() -> str:
+ # Return the latest matched version (e.g. 4.39.1)
+ github_api_url = f"{WEBHOST_GITHUB_API}/tags?page=1&per_page=10"
+ print(f"Checking latest webhost version from {github_api_url}")
+ github_response = urllib.request.urlopen(github_api_url)
+ tags = json.loads(github_response.read())
+
+ # As tags are placed in time desending order, the latest v3
+ # tag should be the first occurance starts with 'v3.' string
+ latest = [gt for gt in tags if gt["name"].startswith(WEBHOST_TAG_PREFIX)]
+ return latest[0]["name"].replace("v", "")
+
+
+def download_webhost_zip(version, branch):
+ with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+ if branch:
+ zip_url = f"{WEBHOST_GIT_REPO}/refs/heads/{branch}.zip"
+ else:
+ zip_url = f"{WEBHOST_GIT_REPO}/v{version}.zip"
+
+ print(f"Downloading Functions Host from {zip_url}")
+ try:
+ urllib.request.urlretrieve(zip_url, temp_file.name)
+ except Exception as e:
+ print(
+ f"Failed to download Functions Host source code from {zip_url}: {e}",
+ file=sys.stderr)
+ sys.exit(1)
+ return temp_file.name
+
+
+def create_webhost_folder(dest_folder):
+ if dest_folder.exists():
+ shutil.rmtree(dest_folder)
+ os.makedirs(dest_folder, exist_ok=True)
+ print(f"Functions Host folder is created in {dest_folder}")
+
+
+def extract_webhost_zip(version, src_zip, dest):
+ print(f"Extracting Functions Host from {src_zip}")
+ with zipfile.ZipFile(src_zip, 'r') as archive:
+ for archive_name in archive.namelist():
+ prefix = f"azure-functions-host-{version}/"
+ if archive_name.startswith(prefix):
+ sanitized_name = archive_name.replace("\\", os.sep).replace(
+ prefix, "")
+ dest_filename = dest / sanitized_name
+ zipinfo = archive.getinfo(archive_name)
+ if not dest_filename.parent.exists():
+ os.makedirs(dest_filename.parent, exist_ok=True)
+ if zipinfo.is_dir():
+ os.makedirs(dest_filename, exist_ok=True)
+ else:
+ with archive.open(archive_name) as src, open(dest_filename,
+ "wb") as dst:
+ dst.write(src.read())
+ print(f"Functions Host is extracted into {dest}")
+
+
+def chmod_protobuf_generation_script(webhost_dir):
+ script_path = webhost_dir / "src" / "WebJobs.Script.Grpc" / "generate_protos.sh"
+ if sys.platform != "win32" and script_path.exists():
+ print("Change generate_protos.sh script permission")
+ os.chmod(script_path, 0o555)
+
+
+def compile_webhost(webhost_dir):
+ print(f"Compiling Functions Host from {webhost_dir}")
+ try:
+ subprocess.run(
+ ["dotnet", "build", "WebJobs.Script.sln", "-o", "bin",
+ "/p:TreatWarningsAsErrors=false"],
+ check=True,
+ cwd=str(webhost_dir),
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ )
+ except subprocess.CalledProcessError:
+ print(
+ f"Failed to compile webhost in {webhost_dir}. "
+ ".NET Core SDK is required to build the solution. "
+ "Please visit https://aka.ms/dotnet-download",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ print("Functions Host is compiled successfully")
+
+
+def gen_grpc():
+ proto_root_dir = ROOT_DIR / "azure_functions_worker" / "protos"
+ proto_src_dir = proto_root_dir / "_src" / "src" / "proto"
+ staging_root_dir = BUILD_DIR / "protos"
+ staging_dir = staging_root_dir / "azure_functions_worker" / "protos"
+ built_protos_dir = BUILD_DIR / "built_protos"
+
+ if os.path.exists(BUILD_DIR):
+ shutil.rmtree(BUILD_DIR)
+
+ shutil.copytree(proto_src_dir, staging_dir)
+ os.makedirs(built_protos_dir)
+
+ protos = [
+ os.sep.join(("shared", "NullableTypes.proto")),
+ os.sep.join(("identity", "ClaimsIdentityRpc.proto")),
+ "FunctionRpc.proto",
+ ]
+
+ for proto in protos:
+ subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "grpc_tools.protoc",
+ "-I",
+ os.sep.join(("azure_functions_worker", "protos")),
+ "--python_out",
+ str(built_protos_dir),
+ "--grpc_python_out",
+ str(built_protos_dir),
+ os.sep.join(("azure_functions_worker", "protos", proto)),
+ ],
+ check=True,
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ cwd=staging_root_dir,
+ )
+
+ compiled_files = glob.glob(
+ str(built_protos_dir / "**" / "*.py"), recursive=True
+ )
+
+ if not compiled_files:
+ print("grpc_tools.protoc produced no Python files", file=sys.stderr)
+ sys.exit(1)
+
+ # Needed to support absolute imports in files. See
+ # https://github.com/protocolbuffers/protobuf/issues/1491
+ make_absolute_imports(compiled_files)
+
+ dir_util.copy_tree(str(built_protos_dir), str(proto_root_dir))
+
+
+def make_absolute_imports(compiled_files):
+ for compiled in compiled_files:
+ with open(compiled, "r+") as f:
+ content = f.read()
+ f.seek(0)
+ # Convert lines of the form:
+ # import xxx_pb2 as xxx__pb2 to
+ # from azure_functions_worker.protos import xxx_pb2 as..
+ p1 = re.sub(
+ r"\nimport (.*?_pb2)",
+ r"\nfrom azure_functions_worker.protos import \g<1>",
+ content,
+ )
+ # Convert lines of the form:
+ # from identity import xxx_pb2 as.. to
+ # from azure_functions_worker.protos.identity import xxx_pb2..
+ p2 = re.sub(
+ r"from ([a-z]*) (import.*_pb2)",
+ r"from azure_functions_worker.protos.\g<1> \g<2>",
+ p1,
+ )
+ f.write(p2)
+ f.truncate()
+
+
+def install_extensions(extensions_dir):
+ if not extensions_dir.exists():
+ os.makedirs(extensions_dir, exist_ok=True)
+
+ if not (extensions_dir / "host.json").exists():
+ with open(extensions_dir / "host.json", "w") as f:
+ f.write("{}")
+
+ if not (extensions_dir / "extensions.csproj").exists():
+ with open(extensions_dir / "extensions.csproj", "w") as f:
+ f.write(EXTENSIONS_CSPROJ_TEMPLATE)
+
+ with open(extensions_dir / "NuGet.config", "w") as f:
+ f.write(NUGET_CONFIG)
+
+ env = os.environ.copy()
+ env["TERM"] = "xterm" # ncurses 6.1 workaround
+ try:
+ subprocess.run(
+ args=["dotnet", "build", "-o", "."],
+ check=True,
+ cwd=str(extensions_dir),
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ env=env,
+ )
+ except subprocess.CalledProcessError:
+ print(
+ ".NET Core SDK is required to build the extensions. "
+ "Please visit https://aka.ms/dotnet-download"
+ )
+ sys.exit(1)
+
+
+@task
+def extensions(c, clean=False, extensions_dir=None):
+ """Build extensions."""
+ extensions_dir = extensions_dir or BUILD_DIR / "extensions"
+ if clean:
+ print(f"Deleting Extensions Directory: {extensions_dir}")
+ shutil.rmtree(extensions_dir, ignore_errors=True)
+ print("Deleted Extensions Directory")
+ return
+
+ print("Installing Extensions")
+ install_extensions(extensions_dir)
+ print("Extensions installed successfully.")
+
+
+@task
+def build_protos(c, clean=False):
+ """Build gRPC bindings."""
+
+ if clean:
+ shutil.rmtree(BUILD_DIR / 'protos')
+ return
+ print("Generating gRPC bindings...")
+ gen_grpc()
+ print("gRPC bindings generated successfully.")
+
+
+@task
+def webhost(c, clean=False, webhost_version=None, webhost_dir=None,
+ branch_name=None):
+ """Builds the webhost"""
+
+ if webhost_dir is None:
+ webhost_dir = BUILD_DIR / "webhost"
+ else:
+ webhost_dir = pathlib.Path(webhost_dir)
+
+ if clean:
+ print("Deleting webhost dir")
+ shutil.rmtree(webhost_dir, ignore_errors=True)
+ print("Deleted webhost dir")
+ return
+
+ if webhost_version is None:
+ webhost_version = get_webhost_version()
+
+ zip_path = download_webhost_zip(webhost_version, branch_name)
+ create_webhost_folder(webhost_dir)
+ version = branch_name or webhost_version
+ extract_webhost_zip(version.replace("/", "-"), zip_path, webhost_dir)
+ chmod_protobuf_generation_script(webhost_dir)
+ compile_webhost(webhost_dir)
+
+
+@task
+def clean(c):
+ """Clean build directory."""
+
+ print("Deleting build directory")
+ shutil.rmtree(BUILD_DIR, ignore_errors=True)
+ print("Deleted build directory")
diff --git a/tests/unittests/load_functions/load_outside_main/main.py b/tests/unittests/load_functions/load_outside_main/main.py
index 9c8dea18..3e5a5e13 100644
--- a/tests/unittests/load_functions/load_outside_main/main.py
+++ b/tests/unittests/load_functions/load_outside_main/main.py
@@ -15,8 +15,6 @@ def main(req: func.HttpRequest):
# Ensure the module can still be loaded from package
from __app__.stub_http_trigger import main # NoQA
- from ..stub_http_trigger import main
-
# Ensure submodules can also be imported
from __app__.stub_http_trigger.stub_tools import FOO # NoQA
diff --git a/tests/unittests/test_file_accessor_factory.py b/tests/unittests/test_file_accessor_factory.py
index 143d09f7..13f8fbad 100644
--- a/tests/unittests/test_file_accessor_factory.py
+++ b/tests/unittests/test_file_accessor_factory.py
@@ -9,10 +9,10 @@ from unittest.mock import patch
from azure_functions_worker.bindings.shared_memory_data_transfer import (
FileAccessorFactory,
)
-from azure_functions_worker.bindings.shared_memory_data_transfer.file_accessor_unix import (
+from azure_functions_worker.bindings.shared_memory_data_transfer.file_accessor_unix import ( # NoQA
FileAccessorUnix,
)
-from azure_functions_worker.bindings.shared_memory_data_transfer.file_accessor_windows import (
+from azure_functions_worker.bindings.shared_memory_data_transfer.file_accessor_windows import ( # NoQA
FileAccessorWindows,
)
diff --git a/tests/utils/constants.py b/tests/utils/constants.py
index 92620c87..34c262f2 100644
--- a/tests/utils/constants.py
+++ b/tests/utils/constants.py
@@ -45,6 +45,23 @@ EXTENSIONS_CSPROJ_TEMPLATE = """\
"""
+NUGET_CONFIG = """\
+
+
+
+
+
+
+
+
+
+
+"""
# PROJECT_ROOT refers to the path to azure-functions-python-worker
# TODO: Find root folder without .parent
diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py
index 85beacd4..aaeb40de 100644
--- a/tests/utils/testutils.py
+++ b/tests/utils/testutils.py
@@ -879,7 +879,7 @@ def popen_webhost(*, stdout, stderr, script_root=FUNCS_PATH, port=None):
' * run the following command from the root folder of',
' the project:',
'',
- f' $ {sys.executable} setup.py webhost',
+ f'cd tests && $ {sys.executable} -m invoke -c test_setup webhost',
'',
' * or download or build the Azure Functions Host and'
' then write the full path to WebHost.dll'
@@ -892,7 +892,7 @@ def popen_webhost(*, stdout, stderr, script_root=FUNCS_PATH, port=None):
' dll = /path/Microsoft.Azure.WebJobs.Script.WebHost.dll',
' * or download Azure Functions Core Tools binaries and',
' then write the full path to func.exe into the ',
- ' `CORE_TOOLS_EXE_PATH` envrionment variable.',
+ ' `CORE_TOOLS_EXE_PATH` environment variable.',
'',
'Setting "export PYAZURE_WEBHOST_DEBUG=true" to get the full',
'stdout and stderr from function host.'