Remove host.json, and the ping function copies, set up dynamically

Instead of copying the ping function along with host.json, link them to the test function app during test setup.
2018-08-28 17:22:05 -04:00 · 2018-08-28 17:22:05 -04:00 · 67c14186d5
--- a/.gitignore
+++ b/.gitignore
@ -103,4 +103,3 @@ ENV/

 .testconfig
 .pytest_cache
-tests/*_functions/bin/
--- a/azure/functions_worker/testutils.py
+++ b/azure/functions_worker/testutils.py
@ -44,6 +44,29 @@ WORKER_PATH = pathlib.Path(__file__).parent.parent.parent
 WORKER_CONFIG = WORKER_PATH / '.testconfig'
 ON_WINDOWS = platform.system() == 'Windows'

+HOST_JSON_TEMPLATE = """\
+{
+    "version": "2.0",
+    "logging": {
+        "logLevel": {
+           "default": "Trace"
+        }
+    },
+    "http": {
+        "routePrefix": "api"
+    },
+    "swagger": {
+        "enabled": true
+    },
+    "eventHub": {
+        "maxBatchSize": 1000,
+        "prefetchCount": 1000,
+        "batchCheckpointFrequency": 1
+    },
+    "functionTimeout": "00:05:00"
+}
+"""
+
 SECRETS_TEMPLATE = """\
 {
  "masterKey": {
@ -131,25 +154,14 @@ class WebHostTestCase(unittest.TestCase, metaclass=WebHostTestCaseMeta):
        else:
            cls.host_stdout = tempfile.NamedTemporaryFile('w+t')

-        extensions = TESTS_ROOT / script_dir / 'bin'
+        _setup_func_app(TESTS_ROOT / script_dir)

-        if not extensions.exists():
-            if extensions.is_symlink():
-                extensions.unlink()
-            elif extensions.exists():
-                shutil.rmtree(str(extensions))
-
-            if ON_WINDOWS:
-                shutil.copytree(EXTENSIONS_PATH, str(extensions))
-            else:
-                extensions.symlink_to(
-                    EXTENSIONS_PATH, target_is_directory=True)
-            cls.linked_extensions = True
-        else:
-            cls.linked_extensions = False
-
-        cls.webhost = start_webhost(script_dir=script_dir,
-                                    stdout=cls.host_stdout)
+        try:
+            cls.webhost = start_webhost(script_dir=script_dir,
+                                        stdout=cls.host_stdout)
+        except Exception:
+            _teardown_func_app(TESTS_ROOT / script_dir)
+            raise

    @classmethod
    def tearDownClass(cls):
@ -160,12 +172,8 @@ class WebHostTestCase(unittest.TestCase, metaclass=WebHostTestCaseMeta):
            cls.host_stdout.close()
            cls.host_stdout = None

-        if cls.linked_extensions:
-            extensions = TESTS_ROOT / cls.get_script_dir() / 'bin'
-            if ON_WINDOWS:
-                shutil.rmtree(extensions)
-            else:
-                extensions.unlink()
+        script_dir = pathlib.Path(cls.get_script_dir())
+        _teardown_func_app(TESTS_ROOT / script_dir)

    def _run_test(self, test, *args, **kwargs):
        if self.host_stdout is None:
@ -598,11 +606,54 @@ def start_webhost(*, script_dir=None, stdout=None):
        time.sleep(0.5)
    else:
        proc.terminate()
+        try:
+            proc.wait(10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
        raise RuntimeError('could not start the webworker')

    return _WebHostProxy(proc, addr)


+def _remove_path(path):
+    if path.is_symlink():
+        path.unlink()
+    elif path.is_dir():
+        shutil.rmtree(str(path))
+    elif path.exists():
+        path.unlink()
+
+
+def _symlink_dir(src, dst):
+    _remove_path(dst)
+
+    if ON_WINDOWS:
+        shutil.copytree(str(src), str(dst))
+    else:
+        dst.symlink_to(src, target_is_directory=True)
+
+
+def _setup_func_app(app_root):
+    extensions = app_root / 'bin'
+    ping_func = app_root / 'ping'
+    host_json = app_root / 'host.json'
+
+    with open(host_json, 'w') as f:
+        f.write(HOST_JSON_TEMPLATE)
+
+    _symlink_dir(TESTS_ROOT / 'ping', ping_func)
+    _symlink_dir(EXTENSIONS_PATH, extensions)
+
+
+def _teardown_func_app(app_root):
+    extensions = app_root / 'bin'
+    ping_func = app_root / 'ping'
+    host_json = app_root / 'host.json'
+
+    for path in (extensions, ping_func, host_json):
+        _remove_path(path)
+
+
 def _main():
    parser = argparse.ArgumentParser(description='Run a Python worker.')
    parser.add_argument('scriptroot',
@ -610,16 +661,8 @@ def _main():

    args = parser.parse_args()

-    extensions = pathlib.Path(args.scriptroot) / 'bin'
-
-    if not extensions.exists():
-        if extensions.is_symlink():
-            extensions.unlink()
-
-        extensions.symlink_to(EXTENSIONS_PATH, target_is_directory=True)
-        linked_extensions = True
-    else:
-        linked_extensions = False
+    app_root = pathlib.Path(args.scriptroot)
+    _setup_func_app(app_root)

    host = popen_webhost(
        stdout=sys.stdout, stderr=sys.stderr,
@ -628,8 +671,7 @@ def _main():
        host.wait()
    finally:
        host.terminate()
-        if linked_extensions:
-            extensions.unlink()
+        _teardown_func_app()


 if __name__ == '__main__':
--- a/setup.py
+++ b/setup.py
@ -15,8 +15,8 @@ from setuptools.command import develop


 # TODO: change this to something more stable when available.
-WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/nheu483rq96ssbst'
-               '/artifacts/Functions.Binaries.2.0.12014-alpha.no-runtime.zip')
+WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/k11e6ngq22n1kgfm'
+               '/artifacts/Functions.Binaries.2.0.12041-alpha.no-runtime.zip')

 # Extensions necessary for non-core bindings.
 AZURE_EXTENSIONS = [
--- a/tests/.gitignore
+++ b/tests/.gitignore
@ -0,0 +1,3 @@
+*_functions/bin/
+*_functions/host.json
+*_functions/ping/
--- a/tests/blob_functions/host.json
+++ b/tests/blob_functions/host.json
@ -1,32 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-        "defaultLevel": "Trace",
-        "categoryLevels": {
-            "Worker": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}
--- a/tests/cosmosdb_functions/host.json
+++ b/tests/cosmosdb_functions/host.json
@ -1,33 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-	"fileLoggingMode": "always",
-        "categoryFilter": {
-            "Worker": "Trace",
-	    "defaultLevel": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}
--- a/tests/cosmosdb_functions/ping/README.md
+++ b/tests/cosmosdb_functions/ping/README.md
@ -1,2 +0,0 @@
-This function is used to check the host availability in tests.
-Please do not remove.
--- a/tests/cosmosdb_functions/ping/function.json
+++ b/tests/cosmosdb_functions/ping/function.json
@ -1,11 +0,0 @@
-{
-  "scriptFile": "main.py",
-  "disabled": false,
-  "bindings": [
-    {
-      "type": "httpTrigger",
-      "direction": "in",
-      "name": "req"
-    }
-  ]
-}
--- a/tests/cosmosdb_functions/ping/main.py
+++ b/tests/cosmosdb_functions/ping/main.py
@ -1,2 +0,0 @@
-def main(req):
-    return
--- a/tests/eventgrid_functions/host.json
+++ b/tests/eventgrid_functions/host.json
@ -1 +0,0 @@
-{}
--- a/tests/eventgrid_functions/ping/README.md
+++ b/tests/eventgrid_functions/ping/README.md
@ -1,2 +0,0 @@
-This function is used to check the host availability in tests.
-Please do not remove.
--- a/tests/eventgrid_functions/ping/function.json
+++ b/tests/eventgrid_functions/ping/function.json
@ -1,11 +0,0 @@
-{
-  "scriptFile": "main.py",
-  "disabled": false,
-  "bindings": [
-    {
-      "type": "httpTrigger",
-      "direction": "in",
-      "name": "req"
-    }
-  ]
-}
--- a/tests/eventgrid_functions/ping/main.py
+++ b/tests/eventgrid_functions/ping/main.py
@ -1,2 +0,0 @@
-def main(req):
-    return
--- a/tests/eventhub_functions/host.json
+++ b/tests/eventhub_functions/host.json
@ -1,32 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-        "defaultLevel": "Trace",
-        "categoryLevels": {
-            "Worker": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}
--- a/tests/eventhub_functions/ping/README.md
+++ b/tests/eventhub_functions/ping/README.md
@ -1,2 +0,0 @@
-This function is used to check the host availability in tests.
-Please do not remove.
--- a/tests/eventhub_functions/ping/function.json
+++ b/tests/eventhub_functions/ping/function.json
@ -1,11 +0,0 @@
-{
-  "scriptFile": "main.py",
-  "disabled": false,
-  "bindings": [
-    {
-      "type": "httpTrigger",
-      "direction": "in",
-      "name": "req"
-    }
-  ]
-}
--- a/tests/eventhub_functions/ping/main.py
+++ b/tests/eventhub_functions/ping/main.py
@ -1,2 +0,0 @@
-def main(req):
-    return
--- a/tests/http_functions/host.json
+++ b/tests/http_functions/host.json
@ -1,32 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-        "defaultLevel": "Trace",
-        "categoryLevels": {
-            "Worker": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}
--- a/tests/http_functions/ping/README.md
+++ b/tests/http_functions/ping/README.md
@ -1,2 +0,0 @@
-This function is used to check the host availability in tests.
-Please do not remove.
--- a/tests/http_functions/ping/function.json
+++ b/tests/http_functions/ping/function.json
@ -1,11 +0,0 @@
-{
-  "scriptFile": "main.py",
-  "disabled": false,
-  "bindings": [
-    {
-      "type": "httpTrigger",
-      "direction": "in",
-      "name": "req"
-    }
-  ]
-}
--- a/tests/http_functions/ping/main.py
+++ b/tests/http_functions/ping/main.py
@ -1,2 +0,0 @@
-def main(req):
-    return
--- a/tests/load_functions/host.json
+++ b/tests/load_functions/host.json
@ -1,32 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-        "defaultLevel": "Trace",
-        "categoryLevels": {
-            "Worker": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}
--- a/tests/load_functions/ping/README.md
+++ b/tests/load_functions/ping/README.md
@ -1,2 +0,0 @@
-This function is used to check the host availability in tests.
-Please do not remove.
--- a/tests/load_functions/ping/function.json
+++ b/tests/load_functions/ping/function.json
@ -1,11 +0,0 @@
-{
-  "scriptFile": "main.py",
-  "disabled": false,
-  "bindings": [
-    {
-      "type": "httpTrigger",
-      "direction": "in",
-      "name": "req"
-    }
-  ]
-}
--- a/tests/load_functions/ping/main.py
+++ b/tests/load_functions/ping/main.py
@ -1,2 +0,0 @@
-def main(req):
-    return
--- a/tests/blob_functions/ping/README.md
+++ b/tests/blob_functions/ping/README.md
--- a/tests/blob_functions/ping/function.json
+++ b/tests/blob_functions/ping/function.json
--- a/tests/blob_functions/ping/main.py
+++ b/tests/blob_functions/ping/main.py
--- a/tests/queue_functions/host.json
+++ b/tests/queue_functions/host.json
@ -1,32 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-        "defaultLevel": "Trace",
-        "categoryLevels": {
-            "Worker": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}
--- a/tests/queue_functions/ping/README.md
+++ b/tests/queue_functions/ping/README.md
@ -1,2 +0,0 @@
-This function is used to check the host availability in tests.
-Please do not remove.
--- a/tests/queue_functions/ping/function.json
+++ b/tests/queue_functions/ping/function.json
@ -1,11 +0,0 @@
-{
-  "scriptFile": "main.py",
-  "disabled": false,
-  "bindings": [
-    {
-      "type": "httpTrigger",
-      "direction": "in",
-      "name": "req"
-    }
-  ]
-}
--- a/tests/queue_functions/ping/main.py
+++ b/tests/queue_functions/ping/main.py
@ -1,2 +0,0 @@
-def main(req):
-    return
--- a/tests/servicebus_functions/host.json
+++ b/tests/servicebus_functions/host.json
@ -1,34 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-        "categoryFilter": {
-            "defaultLevel": "Info",
-        },
-        "categoryLevels": {
-            "Worker": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}
--- a/tests/servicebus_functions/ping/README.md
+++ b/tests/servicebus_functions/ping/README.md
@ -1,2 +0,0 @@
-This function is used to check the host availability in tests.
-Please do not remove.
--- a/tests/servicebus_functions/ping/function.json
+++ b/tests/servicebus_functions/ping/function.json
@ -1,11 +0,0 @@
-{
-  "scriptFile": "main.py",
-  "disabled": false,
-  "bindings": [
-    {
-      "type": "httpTrigger",
-      "direction": "in",
-      "name": "req"
-    }
-  ]
-}
--- a/tests/servicebus_functions/ping/main.py
+++ b/tests/servicebus_functions/ping/main.py
@ -1,2 +0,0 @@
-def main(req):
-    return
--- a/tests/timer_functions/host.json
+++ b/tests/timer_functions/host.json
@ -1,32 +0,0 @@
-{
-    "http": {
-        "routePrefix": "api",
-        "maxConcurrentRequests": 5,
-        "maxOutstandingRequests": 30
-    },
-    "logger": {
-        "defaultLevel": "Trace",
-        "categoryLevels": {
-            "Worker": "Trace"
-        }
-    },
-    "queues": {
-        "visibilityTimeout": "00:00:10"
-    },
-    "swagger": {
-        "enabled": true
-    },
-    "eventHub": {
-        "maxBatchSize": 1000,
-        "prefetchCount": 1000,
-        "batchCheckpointFrequency": 1
-    },
-    "healthMonitor": {
-        "enabled": true,
-        "healthCheckInterval": "00:00:10",
-        "healthCheckWindow": "00:02:00",
-        "healthCheckThreshold": 6,
-        "counterThreshold": 0.80
-    },
-    "functionTimeout": "00:05:00",
-}