From 91f18ffb4854db953114a3f70a61ff8cca3666fb Mon Sep 17 00:00:00 2001 From: gavin-aguiar <80794152+gavin-aguiar@users.noreply.github.com> Date: Mon, 4 Apr 2022 16:54:01 -0500 Subject: [PATCH] Update linux consumption tests to test with dev library (#990) * Update host version for linux consumption tests * Revert host version to v3 * Adding isolate dependencies flag in docker image * Fixed lint tests * Skipping common libraries test for python3.7 --- azure_functions_worker/testutils_lc.py | 39 +++++++++++++++++++----- tests/endtoend/test_linux_consumption.py | 32 ++++++++++++------- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/azure_functions_worker/testutils_lc.py b/azure_functions_worker/testutils_lc.py index 69a79e58..91a105b7 100644 --- a/azure_functions_worker/testutils_lc.py +++ b/azure_functions_worker/testutils_lc.py @@ -1,21 +1,24 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - -from typing import Dict - import base64 import json import os import re +import shutil import subprocess import sys +import tempfile import time import uuid +from io import BytesIO +from typing import Dict +from urllib.request import urlopen +from zipfile import ZipFile +import requests from Crypto.Cipher import AES from Crypto.Hash.SHA256 import SHA256Hash from Crypto.Util.Padding import pad -import requests # Linux Consumption Testing Constants _DOCKER_PATH = "DOCKER_PATH" @@ -23,6 +26,9 @@ _DOCKER_DEFAULT_PATH = "docker" _MESH_IMAGE_URL = "https://mcr.microsoft.com/v2/azure-functions/mesh/tags/list" _MESH_IMAGE_REPO = "mcr.microsoft.com/azure-functions/mesh" _DUMMY_CONT_KEY = "MDEyMzQ1Njc4OUFCQ0RFRjAxMjM0NTY3ODlBQkNERUY=" +_FUNC_GITHUB_ZIP = "https://github.com/Azure/azure-functions-python-library" \ + "/archive/refs/heads/dev.zip" +_FUNC_FILE_NAME = "azure-functions-python-library-dev" class LinuxConsumptionWebHostController: @@ -80,9 +86,9 @@ class LinuxConsumptionWebHostController: f' stdout: {stdout}') def send_request( - self, - req: requests.Request, - ses: requests.Session = None + self, + req: requests.Request, + ses: requests.Session = None ) -> requests.Response: """Send a request with authorization token. Return a Response object""" session = ses @@ -129,6 +135,12 @@ class LinuxConsumptionWebHostController: cls._mesh_images[host_major] = image_tag return image_tag + @staticmethod + def _download_azure_functions() -> str: + with urlopen(_FUNC_GITHUB_ZIP) as zipresp: + with ZipFile(BytesIO(zipresp.read())) as zfile: + zfile.extractall(tempfile.gettempdir()) + def spawn_container(self, image: str, env: Dict[str, str] = {}) -> int: @@ -137,10 +149,18 @@ class LinuxConsumptionWebHostController: """ # Construct environment variables and start the docker container worker_path = os.path.dirname(__file__) + library_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME, + 'azure', 'functions') + self._download_azure_functions() + container_worker_path = ( f"/azure-functions-host/workers/python/{self._py_version}/" "LINUX/X64/azure_functions_worker" ) + container_library_path = ( + f"/azure-functions-host/workers/python/{self._py_version}/" + "LINUX/X64/azure/functions" + ) run_cmd = [] run_cmd.extend([self._docker_cmd, "run", "-p", "0:80", "-d"]) @@ -150,7 +170,9 @@ class LinuxConsumptionWebHostController: run_cmd.extend(["-e", f"CONTAINER_NAME={self._uuid}"]) run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"]) run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"]) + run_cmd.extend(["-e", "PYTHON_ISOLATE_WORKER_DEPENDENCIES=1"]) run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}']) + run_cmd.extend(["-v", f'{library_path}:{container_library_path}']) for key, value in env.items(): run_cmd.extend(["-e", f"{key}={value}"]) @@ -181,7 +203,7 @@ class LinuxConsumptionWebHostController: self._ports[self._uuid] = port_number # Wait for three seconds for the container to be in ready state - time.sleep(3) + time.sleep(6) return port_number def get_container_logs(self) -> str: @@ -259,6 +281,7 @@ class LinuxConsumptionWebHostController: def __exit__(self, exc_type, exc_value, traceback): logs = self.get_container_logs() self.safe_kill_container() + shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME)) if traceback: print(f'Test failed with container logs: {logs}', diff --git a/tests/endtoend/test_linux_consumption.py b/tests/endtoend/test_linux_consumption.py index 6a89cead..1285dc4d 100644 --- a/tests/endtoend/test_linux_consumption.py +++ b/tests/endtoend/test_linux_consumption.py @@ -1,15 +1,16 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from unittest import TestCase, skipIf - import os import sys -from requests import Request +from unittest import TestCase, skipIf from azure_functions_worker.testutils_lc import ( LinuxConsumptionWebHostController ) from azure_functions_worker.utils.common import is_python_version +from requests import Request + +_DEFAULT_HOST_VERSION = "3" @skipIf(is_python_version('3.10'), @@ -42,7 +43,8 @@ class TestLinuxConsumption(TestCase): """In any circumstances, a placeholder container should returns 200 even when it is not specialized. """ - with LinuxConsumptionWebHostController("3", self._py_version) as ctrl: + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: req = Request('GET', ctrl.url) resp = ctrl.send_request(req) self.assertTrue(resp.ok) @@ -51,7 +53,8 @@ class TestLinuxConsumption(TestCase): """An HttpTrigger function app with 'azure-functions' library should return 200. """ - with LinuxConsumptionWebHostController("3", self._py_version) as ctrl: + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url("HttpNoAuth") @@ -60,6 +63,8 @@ class TestLinuxConsumption(TestCase): resp = ctrl.send_request(req) self.assertEqual(resp.status_code, 200) + @skipIf(is_python_version('3.7'), + "Skip the tests for Python 3.7.") def test_common_libraries(self): """A function app with the following requirements.txt: @@ -73,7 +78,8 @@ class TestLinuxConsumption(TestCase): should return 200 after importing all libraries. """ - with LinuxConsumptionWebHostController("3", self._py_version) as ctrl: + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url("CommonLibraries") @@ -98,7 +104,8 @@ class TestLinuxConsumption(TestCase): should return 200 after importing all libraries. """ - with LinuxConsumptionWebHostController("3", self._py_version) as ctrl: + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf") @@ -123,10 +130,11 @@ class TestLinuxConsumption(TestCase): should return 200 after importing all libraries. """ - with LinuxConsumptionWebHostController("3", self._py_version) as ctrl: + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, - "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf") + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf") }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger') resp = ctrl.send_request(req) @@ -144,7 +152,8 @@ class TestLinuxConsumption(TestCase): should return 200 and by default customer debug logging should be disabled. """ - with LinuxConsumptionWebHostController("3", self._py_version) as ctrl: + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url("EnableDebugLogging") @@ -170,7 +179,8 @@ class TestLinuxConsumption(TestCase): should return 200 and with customer debug logging enabled, debug logs should be written to container logs. """ - with LinuxConsumptionWebHostController("3", self._py_version) as ctrl: + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url(