From 797ae7894ecdd14d32753190128c802a8be35658 Mon Sep 17 00:00:00 2001 From: Ray Fang Date: Fri, 20 Jul 2018 01:51:47 +0800 Subject: [PATCH] Add --template param to solution creation (#194) * Use env key instead of value when adding modules * Add CLI entrypoint * Call iotedgedev addmodule when create solution * Add default routes when adding modules * Update template * Update test * Revert changes to monitor timeout * Add temp sensor route when adding module * Add copy_template function * Add default ToIoTHub route when creating solution * Update system module images to GA version * Fix incorrect string escape * Replace image placeholder with real image URL when building images * Enable Node.js module creation * Refind the logic to parse image placeholder * Won't add tempsensor route when adding modules * Minor refinement * Add nested_set utility method * Add default route from temp sensor when solution creation * Rename var_dict to replacement * Use name in env as default when new solution * WIP support for BYPASS_MODULES * WIP support for BYPASS_MODULES * WIP support for BYPASS_MODULES * Update utility methods * Update image_tag_map key type to tuple * Update Docker SDK version and remove iotedgeruntime from requirements.txt * Disable outdated test cases temporarily * Add unit test for deploymentmanifest.py * Add unit test for utility.py * Fix error on Py27 * Compare lists order-insensitively * Fix PyTest failure on Python 3 --- .env.tmp | 8 +- iotedgedev/cli.py | 21 ++- iotedgedev/deploymentmanifest.py | 35 ++++- iotedgedev/dockercls.py | 3 + iotedgedev/envvars.py | 2 +- iotedgedev/module.py | 24 +-- iotedgedev/modules.py | 140 ++++++++++-------- iotedgedev/runtime.py | 1 - iotedgedev/solution.py | 15 +- iotedgedev/template/.env.tmp | 15 +- iotedgedev/template/deployment.template.json | 27 +--- .../template/modules/filtermodule/.gitignore | 31 ---- .../template/modules/filtermodule/Dockerfile | 13 -- .../filtermodule/Dockerfile.amd64.debug | 18 --- .../modules/filtermodule/Dockerfile.arm32v7 | 5 - .../template/modules/filtermodule/Program.cs | 128 ---------------- .../modules/filtermodule/filtermodule.csproj | 22 --- .../template/modules/filtermodule/module.json | 17 --- iotedgedev/template/runtime.template.json | 2 +- iotedgedev/template/template.zip | Bin 8456 -> 3904 bytes iotedgedev/utility.py | 59 ++++++-- requirements.txt | 3 +- setup.cfg | 2 + setup.py | 3 +- tests/assets/deployment.template_1.json | 79 ++++++++++ tests/assets/deployment.template_2.json | 79 ++++++++++ tests/assets/deployment.template_3.json | 90 +++++++++++ tests/conftest.py | 4 + tests/node_solution/.env.tmp | 2 +- tests/test_azurecli.py | 16 +- tests/test_connectionstring.py | 18 ++- tests/test_deploymentmanifest.py | 50 +++++++ tests/test_envvars.py | 10 +- tests/test_iotedgedev.py | 69 +++++---- tests/test_utility.py | 44 ++++++ tests/utility/utility.py | 11 ++ 36 files changed, 651 insertions(+), 415 deletions(-) delete mode 100644 iotedgedev/template/modules/filtermodule/.gitignore delete mode 100644 iotedgedev/template/modules/filtermodule/Dockerfile delete mode 100644 iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug delete mode 100644 iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7 delete mode 100644 iotedgedev/template/modules/filtermodule/Program.cs delete mode 100644 iotedgedev/template/modules/filtermodule/filtermodule.csproj delete mode 100644 iotedgedev/template/modules/filtermodule/module.json create mode 100644 tests/assets/deployment.template_1.json create mode 100644 tests/assets/deployment.template_2.json create mode 100644 tests/assets/deployment.template_3.json create mode 100644 tests/conftest.py create mode 100644 tests/test_deploymentmanifest.py create mode 100644 tests/test_utility.py create mode 100644 tests/utility/utility.py diff --git a/.env.tmp b/.env.tmp index 4ab4188..f0a04e4 100644 --- a/.env.tmp +++ b/.env.tmp @@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="." RUNTIME_HOST_NAME="." # "." - Auto detect -RUNTIME_TAG="1.0-preview" +RUNTIME_TAG="1.0" RUNTIME_VERBOSITY="INFO" # "DEBUG", "INFO", "ERROR", "WARNING" @@ -51,9 +51,9 @@ RUNTIME_LOG_LEVEL="info" # MODULES # -ACTIVE_MODULES="*" - # "*" - to build all modules - # "filtermodule, module1" - Comma delimited list of modules to build +BYPASS_MODULES="" + # "" - to build all modules + # "filtermodule, module1" - Comma delimited list of modules to bypass when building ACTIVE_DOCKER_PLATFORMS="amd64" # "*" - to build all docker files diff --git a/iotedgedev/cli.py b/iotedgedev/cli.py index 29e4d86..aa44fbf 100644 --- a/iotedgedev/cli.py +++ b/iotedgedev/cli.py @@ -61,14 +61,25 @@ def main(set_config, az_cli=None): required=False, help="Creates a new Azure IoT Edge Solution. Use `--create .` to create in current folder. Use `--create TEXT` to create in a subfolder.") @click.argument("name", required=False) -def solution(create, name): +@click.option('--module', + required=False, + default=envvars.get_envvar("DEFAULT_MODULE_NAME", default="filtermodule"), + show_default=True, + help="Specify the name of the default IoT Edge module.") +@click.option("--template", + default="csharp", + show_default=True, + required=False, + type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]), + help="Specify the template used to create the default IoT Edge module.") +def solution(create, name, module, template): utility = Utility(envvars, output) sol = Solution(output, utility) if name: - sol.create(name) + sol.create(name, module, template) elif create: - sol.create(create) + sol.create(create, module, template) @click.command(context_settings=CONTEXT_SETTINGS, help="Creates Solution and Azure Resources") @@ -107,7 +118,7 @@ def e2e(ctx): required=True) @click.option("--template", required=True, - type=click.Choice(["csharp", "python", "csharpfunction"]), + type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]), help="Specify the template used to create the new IoT Edge module.") @click.pass_context def addmodule(ctx, name, template): @@ -444,7 +455,7 @@ def azure(setup, @click.option("--template", default="csharp", required=False, - type=click.Choice(["csharp", "python", "csharpfunction"]), + type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]), help="Specify the template used to create the new IoT Edge module.") @click.option('--build', default=False, diff --git a/iotedgedev/deploymentmanifest.py b/iotedgedev/deploymentmanifest.py index cd2a753..9d5028f 100644 --- a/iotedgedev/deploymentmanifest.py +++ b/iotedgedev/deploymentmanifest.py @@ -10,12 +10,13 @@ import sys class DeploymentManifest: - def __init__(self, envvars, output, path, is_template): + def __init__(self, envvars, output, utility, path, is_template): + self.utility = utility self.output = output try: self.path = path self.is_template = is_template - self.json = json.load(open(path)) + self.json = json.loads(self.utility.get_file_contents(path, expand_env=True)) except FileNotFoundError: self.output.error('Deployment manifest template file "{0}" not found'.format(path)) if is_template: @@ -27,7 +28,7 @@ class DeploymentManifest: envvars.save_envvar("DEPLOYMENT_CONFIG_TEMPLATE_FILE", path) else: self.output.error('Deployment manifest file "{0}" not found'.format(path)) - sys.exit() + sys.exit(1) def add_module_template(self, module_name): """Add a module template to the deployment manifest with amd64 as the default platform""" @@ -37,12 +38,36 @@ class DeploymentManifest: "status": "running", "restartPolicy": "always", "settings": { - "image": \"{MODULES.""" + module_name + """.amd64}\", + "image": \"${MODULES.""" + module_name + """.amd64}\", "createOptions": "" } }""" - self.json["moduleContent"]["$edgeAgent"]["properties.desired"]["modules"][module_name] = json.loads(new_module) + self.utility.nested_set(self.json, ["moduleContent", "$edgeAgent", "properties.desired", "modules", module_name], json.loads(new_module)) + + self.add_default_route(module_name) + + def add_default_route(self, module_name): + """Add a default route to send messages to IoT Hub""" + new_route_name = "{0}ToIoTHub".format(module_name) + new_route = "FROM /messages/modules/{0}/outputs/* INTO $upstream".format(module_name) + + self.utility.nested_set(self.json, ["moduleContent", "$edgeHub", "properties.desired", "routes", new_route_name], new_route) + + def get_modules_to_process(self): + """Get modules to process from deployment manifest template""" + user_modules = self.json.get("moduleContent", {}).get("$edgeAgent", {}).get("properties.desired", {}).get("modules", {}) + modules_to_process = [] + for _, module_info in user_modules.items(): + image = module_info.get("settings", {}).get("image", "") + # If the image is placeholder, e.g., ${MODULES.NodeModule.amd64}, parse module folder and platform from the placeholder + if image.startswith("${") and image.endswith("}") and len(image.split(".")) > 2: + first_dot = image.index(".") + second_dot = image.index(".", first_dot + 1) + module_dir = image[first_dot+1:second_dot] + module_platform = image[second_dot+1:image.index("}")] + modules_to_process.append((module_dir, module_platform)) + return modules_to_process def save(self): """Dump the JSON to the disk""" diff --git a/iotedgedev/dockercls.py b/iotedgedev/dockercls.py index 7cd59a5..147859d 100644 --- a/iotedgedev/dockercls.py +++ b/iotedgedev/dockercls.py @@ -23,6 +23,9 @@ class Docker: self.docker_client = docker.from_env() self.docker_api = docker.APIClient() + def get_os_type(self): + return self.docker_client.info()["OSType"].lower() + def init_registry(self): self.output.header("INITIALIZING CONTAINER REGISTRY") diff --git a/iotedgedev/envvars.py b/iotedgedev/envvars.py index 60b4182..e56f722 100644 --- a/iotedgedev/envvars.py +++ b/iotedgedev/envvars.py @@ -109,7 +109,7 @@ class EnvVars: self.RUNTIME_CONFIG_DIR = self.get_envvar("RUNTIME_CONFIG_DIR", default=".") if self.RUNTIME_CONFIG_DIR == ".": self.set_envvar("RUNTIME_CONFIG_DIR", self.get_runtime_config_dir()) - self.ACTIVE_MODULES = self.get_envvar("ACTIVE_MODULES") + self.BYPASS_MODULES = self.get_envvar("BYPASS_MODULES") self.ACTIVE_DOCKER_PLATFORMS = self.get_envvar("ACTIVE_DOCKER_PLATFORMS", altkeys=["ACTIVE_DOCKER_ARCH"]) self.CONTAINER_REGISTRY_SERVER = self.get_envvar("CONTAINER_REGISTRY_SERVER") self.CONTAINER_REGISTRY_USERNAME = self.get_envvar("CONTAINER_REGISTRY_USERNAME") diff --git a/iotedgedev/module.py b/iotedgedev/module.py index d348c01..f1ec31a 100644 --- a/iotedgedev/module.py +++ b/iotedgedev/module.py @@ -1,6 +1,5 @@ - -import os import json +import os import sys @@ -17,8 +16,7 @@ class Module(object): def load_module_json(self): if os.path.exists(self.module_json_file): try: - self.file_json_content = json.loads( - self.utility.get_file_contents(self.module_json_file)) + self.file_json_content = json.loads(self.utility.get_file_contents(self.module_json_file, expand_env=True)) self.module_language = self.file_json_content.get( "language").lower() @@ -37,15 +35,21 @@ class Module(object): @property def platforms(self): - return self.file_json_content.get("image").get("tag").get("platforms") + return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", "") @property def tag_version(self): - tag = self.file_json_content.get("image").get("tag").get("version") - if tag == "": - tag = "0.0.0" + tag = self.file_json_content.get("image", {}).get("tag", {}).get("version", "0.0.0") return tag - def get_platform_by_key(self, platform): - return self.file_json_content.get("image").get("tag").get("platforms").get(platform) + @property + def repository(self): + return self.file_json_content.get("image", {}).get("repository", "") + + @property + def build_options(self): + return self.file_json_content.get("image", {}).get("buildOptions", []) + + def get_dockerfile_by_platform(self, platform): + return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", {}).get(platform, "") diff --git a/iotedgedev/modules.py b/iotedgedev/modules.py index a2a15d9..d36367e 100644 --- a/iotedgedev/modules.py +++ b/iotedgedev/modules.py @@ -1,17 +1,16 @@ import os import re +import sys from .deploymentmanifest import DeploymentManifest from .dotnet import DotNet from .module import Module -from .modulesprocessorfactory import ModulesProcessorFactory class Modules: def __init__(self, envvars, utility, output, dock): self.envvars = envvars self.utility = utility - self.utility.set_config() self.output = output self.dock = dock self.dock.init_registry() @@ -30,15 +29,15 @@ class Modules: self.output.error("Module \"{0}\" already exists under {1}".format(name, os.path.abspath(self.envvars.MODULES_PATH))) return - deployment_manifest = DeploymentManifest(self.envvars, self.output, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True) + deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True) - repo = "{0}/{1}".format(self.envvars.CONTAINER_REGISTRY_SERVER, name.lower()) + repo = "{0}/{1}".format("${CONTAINER_REGISTRY_SERVER}", name.lower()) if template == "csharp": dotnet = DotNet(self.envvars, self.output, self.utility) dotnet.install_module_template() dotnet.create_custom_module(name, repo, cwd) elif template == "nodejs": - self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator") + self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator", shell=True) cmd = "yo azure-iot-edge-module -n {0} -r {1}".format(name, repo) self.output.header(cmd) self.utility.exe_proc(cmd.split(), shell=True, cwd=cwd) @@ -66,72 +65,97 @@ class Modules: self.build_push(no_build=no_build) def build_push(self, no_build=False, no_push=False): - self.output.header("BUILDING MODULES", suppress=no_build) - # Get all the modules to build as specified in config. - modules_to_process = self.utility.get_active_modules() + bypass_modules = self.utility.get_bypass_modules() + active_platform = self.utility.get_active_docker_platform() + + # map (module name, platform) tuple to tag. + # sample: (('filtermodule', 'amd64'), 'localhost:5000/filtermodule:0.0.1-amd64') + image_tag_map = {} + # map image tag to (module name, dockerfile) tuple + # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ('filtermodule', '/test_solution/modules/filtermodule/Dockerfile.amd64')) + tag_dockerfile_map = {} + # map image tag to build options + # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ["--add-host=github.com:192.30.255.112"]) + tag_build_options_map = {} + # image tags to build + # sample: 'localhost:5000/filtermodule:0.0.1-amd64' + tags_to_build = set() for module in os.listdir(self.envvars.MODULES_PATH): - - if len(modules_to_process) == 0 or modules_to_process[0] == "*" or module in modules_to_process: - + if module not in bypass_modules: module_dir = os.path.join(self.envvars.MODULES_PATH, module) - - self.output.info("BUILDING MODULE: {0}".format(module_dir), suppress=no_build) - module_json = Module(self.output, self.utility, os.path.join(module_dir, "module.json")) - mod_proc = ModulesProcessorFactory(self.envvars, self.utility, self.output, module_dir).get(module_json.language) + for platform in module_json.platforms: + # get the Dockerfile from module.json + dockerfile = os.path.abspath(os.path.join(module_dir, module_json.get_dockerfile_by_platform(platform))) + container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG + tag = "{0}:{1}{2}-{3}".format(module_json.repository, module_json.tag_version, container_tag, platform).lower() + image_tag_map[(module, platform)] = tag + tag_dockerfile_map[tag] = (module, dockerfile) + tag_build_options_map[tag] = module_json.build_options + if len(active_platform) > 0 and (active_platform[0] == "*" or platform in active_platform): + tags_to_build.add(tag) - # build module + deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True) + modules_to_process = deployment_manifest.get_modules_to_process() + + replacements = {} + for module, platform in modules_to_process: + if module not in bypass_modules: + key = (module, platform) + if key in image_tag_map: + tag = image_tag_map.get(key) + tags_to_build.add(tag) + replacements["${{MODULES.{0}.{1}}}".format(module, platform)] = tag + + for tag in tags_to_build: + if tag in tag_dockerfile_map: + module = tag_dockerfile_map.get(tag)[0] + dockerfile = tag_dockerfile_map.get(tag)[1] + self.output.info("BUILDING MODULE: {0}".format(module), suppress=no_build) + self.output.info("PROCESSING DOCKERFILE: {0}".format(dockerfile), suppress=no_build) + self.output.info("BUILDING DOCKER IMAGE: {0}".format(tag), suppress=no_build) + + # BUILD DOCKER IMAGE if not no_build: - if not mod_proc.build(): - continue + # TODO: apply build options + build_options = self.filter_build_options(tag_build_options_map.get(tag, None)) - docker_arch_process = [docker_arch.strip() for docker_arch in self.envvars.ACTIVE_DOCKER_PLATFORMS.split(",") if docker_arch] + context_path = os.path.abspath(os.path.join(self.envvars.MODULES_PATH, module)) + dockerfile_relative = os.path.relpath(dockerfile, context_path) + # a hack to workaround Python Docker SDK's bug with Linux container mode on Windows + if self.dock.get_os_type() == "linux" and sys.platform == "win32": + dockerfile = dockerfile.replace("\\", "/") + dockerfile_relative = dockerfile_relative.replace("\\", "/") - for arch in module_json.platforms: - if len(docker_arch_process) == 0 or docker_arch_process[0] == "*" or arch in docker_arch_process: + build_result = self.dock.docker_client.images.build(tag=tag, path=context_path, dockerfile=dockerfile_relative) - # get the docker file from module.json - docker_file = module_json.get_platform_by_key(arch) + self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result)) - self.output.info("PROCESSING DOCKER FILE: " + docker_file, suppress=no_build) + if not no_push: + # PUSH TO CONTAINER REGISTRY + self.output.info("PUSHING DOCKER IMAGE: " + tag) - docker_file_name = os.path.basename(docker_file) - container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG - tag_name = module_json.tag_version + container_tag + for line in self.dock.docker_client.images.push(repository=tag, stream=True, auth_config={ + "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}): + self.output.procout(self.utility.decode(line).replace("\\u003e", ">")) + self.output.footer("BUILD COMPLETE", suppress=no_build) + self.output.footer("PUSH COMPLETE", suppress=no_push) + self.utility.set_config(force=True, replacements=replacements) - # publish module - if not no_build: - self.output.info("PUBLISHING MODULE: " + module_dir) - mod_proc.publish() + @staticmethod + def filter_build_options(build_options): + """Remove build options which will be ignored""" + if build_options is None: + return None - image_destination_name = "{0}/{1}:{2}-{3}".format(self.envvars.CONTAINER_REGISTRY_SERVER, module, tag_name, arch).lower() + filtered_build_options = [] + for build_option in build_options: + build_option = build_option.strip() + parsed_option = re.compile(r"\s+").split(build_option) + if parsed_option and ["--rm", "--tag", "-t", "--file", "-f"].index(parsed_option[0]) < 0: + filtered_build_options.append(build_option) - self.output.info("BUILDING DOCKER IMAGE: " + image_destination_name, suppress=no_build) - - # cd to the module folder to build the docker image - project_dir = os.getcwd() - os.chdir(os.path.join(project_dir, module_dir)) - - # BUILD DOCKER IMAGE - - if not no_build: - build_result = self.dock.docker_client.images.build(tag=image_destination_name, path=".", dockerfile=docker_file_name, buildargs={"EXE_DIR": mod_proc.exe_dir}) - - self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result)) - - # CD BACK UP - os.chdir(project_dir) - - if not no_push: - # PUSH TO CONTAINER REGISTRY - self.output.info("PUSHING DOCKER IMAGE TO: " + image_destination_name) - - for line in self.dock.docker_client.images.push(repository=image_destination_name, stream=True, auth_config={ - "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}): - self.output.procout(self.utility.decode(line).replace("\\u003e", ">")) - - self.output.footer("BUILD COMPLETE", suppress=no_build) - self.output.footer("PUSH COMPLETE", suppress=no_push) + return filtered_build_options diff --git a/iotedgedev/runtime.py b/iotedgedev/runtime.py index e45b834..3bc7f3e 100644 --- a/iotedgedev/runtime.py +++ b/iotedgedev/runtime.py @@ -32,4 +32,3 @@ class Runtime: self.dock.remove_modules() self.setup() self.start() - diff --git a/iotedgedev/solution.py b/iotedgedev/solution.py index 43bba9a..3e945b6 100644 --- a/iotedgedev/solution.py +++ b/iotedgedev/solution.py @@ -1,12 +1,13 @@ import os import zipfile + class Solution: def __init__(self, output, utility): self.output = output self.utility = utility - def create(self, name): + def create(self, name, module, template): if name == ".": dir_path = os.getcwd() else: @@ -25,13 +26,17 @@ class Solution: self.output.error("Error while trying to load template.zip") self.output.error(str(ex)) - if name == ".": - name = "" - zipf = zipfile.ZipFile(template_zip) zipf.extractall(name) + self.utility.copy_template(os.path.join(dir_path, "deployment.template.json"), None, {"%MODULE%": module}, False) + os.rename(os.path.join(name, ".env.tmp"), os.path.join(name, ".env")) + + mod_cmd = "iotedgedev addmodule {0} --template {1}".format(module, template) + self.output.header(mod_cmd) + self.utility.call_proc(mod_cmd.split(), cwd=name) + self.output.footer("Azure IoT Edge Solution Created") - if name != "": + if name != ".": self.output.info("Execute 'cd {0}' to navigate to your new solution.".format(name)) diff --git a/iotedgedev/template/.env.tmp b/iotedgedev/template/.env.tmp index 4ab4188..e8a4020 100644 --- a/iotedgedev/template/.env.tmp +++ b/iotedgedev/template/.env.tmp @@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="." RUNTIME_HOST_NAME="." # "." - Auto detect -RUNTIME_TAG="1.0-preview" +RUNTIME_TAG="1.0" RUNTIME_VERBOSITY="INFO" # "DEBUG", "INFO", "ERROR", "WARNING" @@ -51,13 +51,14 @@ RUNTIME_LOG_LEVEL="info" # MODULES # -ACTIVE_MODULES="*" - # "*" - to build all modules - # "filtermodule, module1" - Comma delimited list of modules to build +BYPASS_MODULES="" + # "" - to build all modules + # "filtermodule, module1" - Comma delimited list of modules to bypass when building -ACTIVE_DOCKER_PLATFORMS="amd64" - # "*" - to build all docker files - # "amd64,amd64.debug" - Comma delimted list of docker files to build +ACTIVE_DOCKER_PLATFORMS="" + # "" - to only build Dockerfiles specified in DEPLOYMENT_CONFIG_TEMPLATE_FILE + # "*" - to build all Dockerfiles + # "amd64,amd64.debug" - Comma delimited list of Dockerfiles to build CONTAINER_TAG="" diff --git a/iotedgedev/template/deployment.template.json b/iotedgedev/template/deployment.template.json index 2946e7a..32bcbf1 100644 --- a/iotedgedev/template/deployment.template.json +++ b/iotedgedev/template/deployment.template.json @@ -14,7 +14,7 @@ "edgeAgent": { "type": "docker", "settings": { - "image": "microsoft/azureiotedge-agent:${RUNTIME_TAG}", + "image": "mcr.microsoft.com/azureiotedge-agent:${RUNTIME_TAG}", "createOptions": "" } }, @@ -23,8 +23,8 @@ "status": "running", "restartPolicy": "always", "settings": { - "image": "microsoft/azureiotedge-hub:${RUNTIME_TAG}", - "createOptions": "" + "image": "mcr.microsoft.com/azureiotedge-hub:${RUNTIME_TAG}", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" } } }, @@ -35,17 +35,7 @@ "status": "running", "restartPolicy": "always", "settings": { - "image": "microsoft/azureiotedge-simulated-temperature-sensor:${RUNTIME_TAG}", - "createOptions": "" - } - }, - "filtermodule": { - "version": "1.0", - "type": "docker", - "status": "running", - "restartPolicy": "always", - "settings": { - "image": "${CONTAINER_REGISTRY_SERVER}/filtermodule:0.0.1-amd64", + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:${RUNTIME_TAG}", "createOptions": "" } } @@ -56,19 +46,12 @@ "properties.desired": { "schemaVersion": "1.0", "routes": { - "sensorToFilter": "FROM /messages/modules/temp-sensor-module/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/filtermodule/inputs/input1\")", - "filterToIoTHub": "FROM /messages/modules/filtermodule/outputs/output1 INTO $upstream" + "sensorTo%MODULE%": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/%MODULE%/inputs/input1\")" }, "storeAndForwardConfiguration": { "timeToLiveSecs": 7200 } } - }, - "filtermodule": { - "properties.desired": { - "schemaVersion": "1.0", - "TemperatureThreshold": 21 - } } } } \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/.gitignore b/iotedgedev/template/modules/filtermodule/.gitignore deleted file mode 100644 index 39a824a..0000000 --- a/iotedgedev/template/modules/filtermodule/.gitignore +++ /dev/null @@ -1,31 +0,0 @@ -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ -**/Properties/launchSettings.json - -*_i.c -*_p.c -*_i.h -*.ilk -*.meta -*.obj -*.pch -*.pdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc -.vs \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Dockerfile b/iotedgedev/template/modules/filtermodule/Dockerfile deleted file mode 100644 index 3df26fa..0000000 --- a/iotedgedev/template/modules/filtermodule/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM microsoft/dotnet:2.0-sdk AS build-env -WORKDIR /app - -COPY *.csproj ./ -RUN dotnet restore - -COPY . ./ -RUN dotnet publish -c Release -o out - -FROM microsoft/dotnet:2.0-runtime -WORKDIR /app -COPY --from=build-env /app/out ./ -ENTRYPOINT ["dotnet", "filtermodule.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug b/iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug deleted file mode 100644 index d003a0d..0000000 --- a/iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug +++ /dev/null @@ -1,18 +0,0 @@ -FROM microsoft/dotnet:2.0-sdk AS build-env -WORKDIR /app - -COPY *.csproj ./ -RUN dotnet restore - -COPY . ./ -RUN dotnet publish -c Debug -o out - -FROM microsoft/dotnet:2.0-runtime-stretch -WORKDIR /app -COPY --from=build-env /app/out ./ - -RUN apt-get update -RUN apt-get install -y unzip procps -RUN curl -sSL https://aka.ms/getvsdbgsh | bash /dev/stdin -v latest -l ~/vsdbg - -ENTRYPOINT ["dotnet", "filtermodule.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7 b/iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7 deleted file mode 100644 index d596817..0000000 --- a/iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7 +++ /dev/null @@ -1,5 +0,0 @@ -FROM microsoft/dotnet:2.0.0-runtime-stretch-arm32v7 -ARG EXE_DIR=. -WORKDIR /app -COPY $EXE_DIR/ ./ -CMD ["dotnet", "filtermodule.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Program.cs b/iotedgedev/template/modules/filtermodule/Program.cs deleted file mode 100644 index b5f565b..0000000 --- a/iotedgedev/template/modules/filtermodule/Program.cs +++ /dev/null @@ -1,128 +0,0 @@ -namespace filtermodule -{ - using System; - using System.IO; - using System.Runtime.InteropServices; - using System.Runtime.Loader; - using System.Security.Cryptography.X509Certificates; - using System.Text; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Azure.Devices.Client; - using Microsoft.Azure.Devices.Client.Transport.Mqtt; - - class Program - { - static int counter; - - static void Main(string[] args) - { - // The Edge runtime gives us the connection string we need -- it is injected as an environment variable - string connectionString = Environment.GetEnvironmentVariable("EdgeHubConnectionString"); - - // Cert verification is not yet fully functional when using Windows OS for the container - bool bypassCertVerification = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - if (!bypassCertVerification) InstallCert(); - Init(connectionString, bypassCertVerification).Wait(); - - // Wait until the app unloads or is cancelled - var cts = new CancellationTokenSource(); - AssemblyLoadContext.Default.Unloading += (ctx) => cts.Cancel(); - Console.CancelKeyPress += (sender, cpe) => cts.Cancel(); - WhenCancelled(cts.Token).Wait(); - } - - /// - /// Handles cleanup operations when app is cancelled or unloads - /// - public static Task WhenCancelled(CancellationToken cancellationToken) - { - var tcs = new TaskCompletionSource(); - cancellationToken.Register(s => ((TaskCompletionSource)s).SetResult(true), tcs); - return tcs.Task; - } - - /// - /// Add certificate in local cert store for use by client for secure connection to IoT Edge runtime - /// - static void InstallCert() - { - string certPath = Environment.GetEnvironmentVariable("EdgeModuleCACertificateFile"); - if (string.IsNullOrWhiteSpace(certPath)) - { - // We cannot proceed further without a proper cert file - Console.WriteLine($"Missing path to certificate collection file: {certPath}"); - throw new InvalidOperationException("Missing path to certificate file."); - } - else if (!File.Exists(certPath)) - { - // We cannot proceed further without a proper cert file - Console.WriteLine($"Missing path to certificate collection file: {certPath}"); - throw new InvalidOperationException("Missing certificate file."); - } - X509Store store = new X509Store(StoreName.Root, StoreLocation.CurrentUser); - store.Open(OpenFlags.ReadWrite); - store.Add(new X509Certificate2(X509Certificate2.CreateFromCertFile(certPath))); - Console.WriteLine("Added Cert: " + certPath); - store.Close(); - } - - - /// - /// Initializes the DeviceClient and sets up the callback to receive - /// messages containing temperature information - /// - static async Task Init(string connectionString, bool bypassCertVerification = false) - { - Console.WriteLine("Connection String {0}", connectionString); - - MqttTransportSettings mqttSetting = new MqttTransportSettings(TransportType.Mqtt_Tcp_Only); - // During dev you might want to bypass the cert verification. It is highly recommended to verify certs systematically in production - if (bypassCertVerification) - { - mqttSetting.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; - } - ITransportSettings[] settings = { mqttSetting }; - - // Open a connection to the Edge runtime - DeviceClient ioTHubModuleClient = DeviceClient.CreateFromConnectionString(connectionString, settings); - await ioTHubModuleClient.OpenAsync(); - Console.WriteLine("IoT Hub module client initialized."); - - // Register callback to be called when a message is received by the module - await ioTHubModuleClient.SetInputMessageHandlerAsync("input1", PipeMessage, ioTHubModuleClient); - } - - /// - /// This method is called whenever the module is sent a message from the EdgeHub. - /// It just pipe the messages without any change. - /// It prints all the incoming messages. - /// - static async Task PipeMessage(Message message, object userContext) - { - int counterValue = Interlocked.Increment(ref counter); - - var deviceClient = userContext as DeviceClient; - if (deviceClient == null) - { - throw new InvalidOperationException("UserContext doesn't contain " + "expected values"); - } - - byte[] messageBytes = message.GetBytes(); - string messageString = Encoding.UTF8.GetString(messageBytes); - Console.WriteLine($"Received message: {counterValue}, Body: [{messageString}]"); - - if (!string.IsNullOrEmpty(messageString)) - { - var pipeMessage = new Message(messageBytes); - foreach (var prop in message.Properties) - { - pipeMessage.Properties.Add(prop.Key, prop.Value); - } - await deviceClient.SendEventAsync("output1", pipeMessage); - Console.WriteLine("Received message sent"); - } - return MessageResponse.Completed; - } - } -} diff --git a/iotedgedev/template/modules/filtermodule/filtermodule.csproj b/iotedgedev/template/modules/filtermodule/filtermodule.csproj deleted file mode 100644 index ecab4e5..0000000 --- a/iotedgedev/template/modules/filtermodule/filtermodule.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - Exe - netcoreapp2.0 - - - - True - - - - - - - - - - - - - - \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/module.json b/iotedgedev/template/modules/filtermodule/module.json deleted file mode 100644 index 3d1b2f6..0000000 --- a/iotedgedev/template/modules/filtermodule/module.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema-version": "0.0.1", - "description": "", - "image": { - "repository": "/", - "tag": { - "version": "0.0.1", - "platforms": { - "amd64": "./Dockerfile", - "amd64.debug": "./Dockerfile.amd64.debug", - "arm32v7": "./Dockerfile.arm32v7", - "windows-amd64": "./Dockerfile" - } - } - }, - "language": "csharp" -} \ No newline at end of file diff --git a/iotedgedev/template/runtime.template.json b/iotedgedev/template/runtime.template.json index 38ae19a..6f30624 100644 --- a/iotedgedev/template/runtime.template.json +++ b/iotedgedev/template/runtime.template.json @@ -1,7 +1,7 @@ { "deployment": { "docker": { - "edgeRuntimeImage": "microsoft/azureiotedge-agent:${RUNTIME_TAG}", + "edgeRuntimeImage": "mcr.microsoft.com/azureiotedge-agent:${RUNTIME_TAG}", "loggingOptions": { "log-driver": "json-file", "log-opts": { diff --git a/iotedgedev/template/template.zip b/iotedgedev/template/template.zip index 459f1e9ba34ba258ddc6be5c9ba4883ce3af12a9..545faec71d60fdb92799eaeec511ad4563e552f4 100644 GIT binary patch delta 3097 zcmaJ@c{r47AAZL^_83iL9b5K&#GtZdok=*hgEF#}EHPwn5VBK)LKwR&WnT(II7rE! zB}-XC7)wqV^y!@Abk28u*Y`Ys-0$@~&+oqP>sfy9FG{nO7iWZ`q=EndKnG9+Z=rX( z8(hq(0id7p#|QvOJG$SI_IA5nYq6X#B+Gh~*r95vaiH<*YTbGH z!)Vfku%~Fm)_UBU4NB}0m1g|fW{?6eP9dw=^KgO*@zn;(qSb{pX5CLvWqxK$*|u{l z0eT_65BE7f$bFu&*W}Zw7VpU2{DxP|Ze5)hwl>E=dnhUO4H6A~adO;quNnw!B33}F za`vIDp+fQiN3UZYG2Tm@>#;wz0!sxBTwwxlEDT91yiSWrl-P)l(vLY{Ta>FZsjWEO z7!*XDAz2sS6KK7?Cbs@|74EmbC)g1|)yv0_iW|tc<{!6>iOaQHu;SWoo-LI&-ctS| zZ#SiLC!JwoOU{p4iEWcs-QP;m0C-p!j#}tPMQI4zLK#fw5pxb&FX^!0TV!^xj)v&= zcNAvGFfaCX_qDvy;tyX+R8?W0+l+TH+c&M%L$hB?*t^B3?SxebJ>hH2QRv(*;ysAt z##Kzzu2jg;RYAtDT}AHNHGN}{hEZX(KRQL_gRQEP7o#?)IuUsP0%ko{%u8S)+(7*W z(~Pi~Nm}+|k`Hz<(0(Q>VYR+3_D zP_b*;JrLn45^|@=#jd`BeZQ&)f0XKqE1287n1^|DfM7QLjhF~>-9Oyp&|OToe7l** zGBir(R;=Yb)iQF8N+3Hhl)Z)9G9mDzqMl$7Z#;kCPSyVPnQuXge(q%H;Tj29b`Q1e zS(F;gGEPRD65p>~Ng>tzK{jY5J{SC~WMvPPLcQ(fiBVixQNI0HyKRt4!%UqE_W-gs zdGBSoJjUGP(V#+sA|!(;R5ahU-IOh#u^XPs5u!8Yp`VpEmF{H^dS=;ZueIC~?z$cM z4*rWXyK2+M!ScHUHnl0Pkt&)r)82Z76N<(hrYoqH;U`lx#w5`f=~tq@dGIQ2mUC+~ z(b-LF^kw&1|J$UxqqUW>qfcdSxDd`ggUdrtLfV(=dl{~B*MT=HFOa(4Q|E0-QSBlj$JB=Mn5VGV41f7u4wO? zHmj6!o1?2tV}(nlrgB>6C~DiXkSp?(%G<3khAsw(Oq<=A&Ef zk(m?g#V%%O){(sQ$B1ZL=%XZII(mEw3VayO#{RpZE;2HlbKzFy&@UBsHhhL3LwkjL zgjO;I6{3{%AKfin0y7$sw5ap#T7D( zW4W;MG{h%-)#5?ob2eS9LXjCQ$Shprx0ox4UQ5l;M=%7xQJTvgT+6KpG}JeWT@*U$ zt7zfV9jx}Ml{S!Kr_jX-@zz%WC0R?&Nk8S-N2umjQ3%|^$LpCe^CVBRxNZpWL4Iq2 zz9pEltQq0MApO-3l1~|U`4>Akiq2cfNXc(S6a?Ij0*ip7#Vj)3yje;?#c54g-~}#< z!g}oVnK14be3?nOp41Hc&^n?_p(|oN=y012p0;(qi&6u z3})RVs|Ywk`P5=*Mmxv(L%#bmw_TNrbrMXeHL9)%L{_{iBiv_rxM(VZX3eD4nn_!p zo|d~j$+B6V(Tw-3FChx1H^=J+#OT_qDdEtj=2jj3+k&SdOCu7~Us6av)9^Z`4sS}0 zq5OgM)Pg^-dWw1;#0>YEL;pVZ!Ge_n048Yg?`U~(BWA`P6SAy#vunPy{%SfQ1n>9Z z{1pGpyWG@ZWUyIsX#VW`VevXavSz2z5xGwvE(!wYtl7y_27jB%B$8a6caAbY%RSBW zX11->H0qAL<}Mt_kb6dFGgVGWL`MW;6tbp+i3q*1CJj0l1nME)tJIX(s=Qm@HDL|l z#UX6CQ9CYGX?ACa2h6r@5Pmarb%=P~j;OABajcp{>0<88`9qt+;v_6jSF%GoysfOg z((RJDRMqe2@(Q$*@HQfoV|TO)DaF>#njs_RJ$WUyhEfC&6D!WEkyiT$zAoxFybDx6 zjPTZ(e@m>d{?fDi-u0{0q-n`u1dXz0WH0NK@rK_bMOR3XSn*X%}#5K5Y$fX7q_fi#?yw^61Pqfo4<#P<7 z9H`o$mL!iekByJ>CSw-}lD#XL$oaU}=Zo;n?t_(tvU8af)p@b86OM<^Gh8FkpPww^ zc)EBPb-hyAxCaikGBk!;Z()bk`m2S zSd_J^;ADPA{4PoH>}(JMTA@a2hP`V-1|K2P{E)*b%JQ=l8CO=#e8sx?lG-Dh!_UQaxLcRX=^x=N=}#;eh!BP~=Qe%-t~o;$6EehPw7jAA_v$ zhvHIV-Ub(SH{wP6u;7w=epiYdA%(HJ6_ppwQ;F-!yUOu4cODKt5-gR~|nqxrDm z2y)Ka;X)~#*Z9&#SgMk;1evW&YH(SMC*iVJn0~rX`LbuT#kt!dE1bCc=+SYSwf;p{ z^pywaWt%9UzG*KDrU6lR)EM*0jE08{*AS>~8CmpRN8eoLDHUh4^6D<{n+1bG7MvC- zjYRm1Md?&Sa_1Rb>`u$j%WHr=Yc+o&;hVYd&W8o5a3R*1wej+)z#3N!K?Tj|RIZWk zKw~m%(Dt~Cdq4ZI<~t`~gaT1q1pY_mhkwT@Cej~`jdtN32IUY)Tl?I@AT%A$F>G@P z^v|dt35t)43lxM`<$~eN%>P`WEAwys836#t5$NNh^6y3m2>tcVdWLOoB?hI|H*iQCrUq=>40_IG4+~#aDUzL4QHkT0QSGBe-KgA{G@iZ^KrMo zA?1z40@x5o!*hL)6;uAvlZq`OlQq&uXfOOS4m5*mC$T9iErPJI|i=d)Ho@HEaL3vK%}D9smG90njNn%iCvlEKwl>0P^VH-vIzXW_GS1 z7h8u~&Ck)}?AYE?%M_)Q`q&Chv%%`&qwY;OwyJAYdIajTU7zo%Hql_3Wnc4~&?~CB zCZ>vO(AM=jn{x_rW#~S#pB2q*J3C-Fy+a}4xN)(vv5tW~`)&#GK<3b^C2_?1VzerL zs{8dkiUBkCROQMAk2n?iGfLk#h$uew%HsK^UQcqIRkDC8_zd`ZxIJKhv_{i>7SLW! z+<@Tw*|zu8q#1Wg@ubTbC|BQ3-I-E@7s6reSD9)PkkBk6(9DTfqQFFg2c9V3A!_03 zzDd3-!4^0wp(kB&U1W?nWD~gR^#<*^WKS(<9l1S`Tjj8(BDrGkK0RBo(*0NmPYe$BLyg+U8Ztn|$&Zpra@tJ|lF>xvD-If2k<^8yowoHtRv0T8 zZ-#efc(B-lvA7NFfea4$vqamft2ANDAr3~%z4Le@N_X5F$hB+VaND2UFiZYE@VZ)Us z_SbGy{(5{Ca%3^zh=XU|Cpczt-gu)Mj_I?bwQ+^~?W z*VIA@@*x-3zxK5BxOJ?Mayd9MMSoJ7I6)F>gk9V&M}M4Mw+rUSm^BShD00aHWGs7B z^VIBV_xTk&DoMDhR0@d2?pm@|6+dHgkc^!_*_xB|%&U z&2iOp2pj`Szn$(1j|p@@Yl(%NZup-7-8i4BU?!_t&rK-t=SEmvyc8yL!vyGT1bV7w zh6W!u<8%o{H-&yJI@ue4(p~64Nx~8_^&l5!m&Ili zCZ~^Cv6OGS`+~;&885a1Uc3$W*^Y>>8f)8yhV>(Tm6xc6D#*u*p0?6p)x4rM1%Mg~ z>bY@!S6G0IszCuyG=Dw#Wv2-4NL<5dc|RE>VVJIcoAK1-5>)8-{hvNn}myaX@V6O5gc?E?$`8J7%>3l8LGeLY|TcU%bt`fYm{z=L~ z*m#HXW@ejIR7G7w!Z5 z0@8xY{=2ny;liAZJ%cs}ppysTmSbajr$^%b*GhxqnsN^B%4Se_?c=a1aHX89^M-aY z{B)5RnT5u442co-2}S{rPb{Y|TJCZ_)uvL+(zBhf*PEcis*Lc*8b*)}0@FMgs?X_T zsYL`i&n*pQm5_LFSvYk|(lCbZN$ewYBgmkjt*}d*AIY#g87rWvinNM5Nis05m8B=- z#4$J5F!X7?+P(nMrkT>@>)94t)aqAv@#HlptFI~cOQXtJ0gf&q_GkV!C=05}AE`0g zCEy53L~Seu=Q}M|9=e{WC@sF3n1JUl)XGI$)~tQuVlmSaEj=zG#TJ(h@v4bf^tC6P z`&4DRUR)ZS5_~hBjZcU_NA~hv9-mRps>cJg=H$JS=0>AS#unRLi0v%vc<27vp5OkE~xWMBr|1-}uWaWR#sC&7K{4zNL6F;;ks1vv)G1 z?UO?TP@yA?d5alz1v|ffFU2WcrR?k@@#rK=wbAK;F{71k;x`^XDzR;`)lUIob6fdu zoSVFL!d4J|A^?DiFmrjR*MDArRccpf6MIuLwx6R={u;Gw;yqdi2LNCq004KP<3WFq zvN3|#nOK6XobBy`Rfg>Da$vV;ZSia4Ajb~aiDRk1z!uHM=S9%KRxCs=7i{8~K55`` z{lKJ?LB`*dU6YD0q+QpNnS5dH>qEEP?|cv_!crRA$i2|s6CO5Kd z*}nU$FZkwBDnXI?BY4~%Z%d-)Rw>49k3&laH`gYvhMjSBmn=Dx!}}jROH`mI6wDfZ zRex##o`b`FJF84x~2v$8=HW^b?Aim5XCZi^g z;z`^~bf!3k!%v0W4m(bL*WwexG8hbB0mYv%+w0Bom=t#XQ^ZnoY4l@sOZu_j@;?kx z1eW^QA@cp7YDKRmZYck^Iea@eU!C$n9WR3ALLwy$cxHnG#(OnU2U>DAz{NTYu zmVA18!&a}f+z^on+p~B4!q(paztX&CSSRnF7AMDc2LMQg zt^^`znoZ3dZ0tR3&Fox2E@rk4HbyRHf8}~kVL{IF#NP!PlH|+$!TaP{MH&ZJ%G zWRmucg~MjApEbnTuDB)qywKE!RBFj)*_~H9Tvn!>oTpsSkX(|bri1!(X!VEH$k$`< zFKEuG4GZAp&=iwBo%i>0$xwDw%sbR0eD_-*N;PB!JnJ|EhpBkXi$+c-TUv||7b<)L z7Ppb+$IpR8Ri<6tcX3dKCgoDBMC2j5Gs(RVmkl=Wtz(mbh4s&_*a{uEMu1wpRrz6> zJnYOF$wZ%gs1rRIeO+7sS!4h}89?@5701@z6k=oM{7=RCcU{7VF2$dt%)vG;W==ou z{`m+BzzynnEGwm{06LTjTBs=g^pRgBRAWR0t&VacWRkX=u9b#+-ePWEtk>?Vh{5`C6>QR+D3z z{upwy6fXUs67nUh8mFRB$(Ii{MA%enihP}sbKzoiik_7?a4tqTh=>WnVMHi+FQhfq zqBSLh{DM5L;fDXfTHiH(?n>`4Ww)W`02)?$wq z9f`uc@37uM*O5YMBLBwvQD}TX-+zr#d8PY4`<=r*!HaS;uQm1X1@26&9yGDF1~t2? z^I)5nFa3(7k2X1{+ZnCGRVZ>q!J!2-n-86eATB<4ewC!}Nu9hLRltTu`x<&N|0$`z zA_f`RnsPq?nVK0xEFdu=a-EN{gZKTfL#d=3tCuSmMM_0gQsXr9!a@~XQ%mPe6Vs>t z?%%rBhvP9FKVUyvZ{8P-@8)u8JSEtTZm9XlobvRG7Wx{w-6v&4TN<^SSh{kF3m${` zwqR#)9GzRY6_C&)YWOo&pGQhOn_Lo08#B%Jgi7K!!UkpHDAtO5>&0jo>_G0(r6qxQ zEWu4h4w~{pJEi2?IHJ!UpXW&1dr4v@soT3KNh|3@5$F+%#?$v+1aY(pQ5_r1h?Hva zr-?(p9t3RDTZWSk^AJ9J;nuhE#jncBIM=>VoK&yapIT-Po}{gdl*SOr=bS+xYnJWN znyUuOD-!p?f|O}mQ+`|Ni{Sx7bx>6Qyf56ZiM2$Nl}CM6+UyUV&`{__|Nog-CtEH~ z*GH_8BESH4qR0DzH=!|OmA1yAlpEM!9IA+?O!q}+^V-)Aggg@XNwP0Kq)qnF?6;z^H*2c#|-L7r?3Z<)Q*u}$aB;z=iV zQA=p$u$@#j;MllZr|)TR3(Vl<5I5uZJUhLG{}sjehc?ujE!6ueDEJx_C=7xm)!}-${5qcnn1os|YXm-v z(RQ(BOz-Rrdp_OIz6KPex0}v5Wb%p?ndbG&@z4-B1#UC<_6Kr4ZuWW%u6k(Nu9eG` z8#;&{-~b1)ce@QIQjHXGi!ESJ^3Cl4*)>m}-$7XQ3T5wA?$E=TiCA*?*<2f0lMOKhlmP#6d=`$oQxqCUyWSGFO)Y%pJg5|i3awmP0wBl?aOFix#HH!b|ESZ z=ZU7670R8{x5?u#cQU<%YSRZyxhfoXh9JqAB?ccy zjS3(h(rhlhx9i6^nA>gKwbz`x@;MHGscGLk&{#DJ_;zUOK0?z-`KJ{ADENOhAGWbW zm;vnALHl$qfnWjI_DoA-6ncZa4%K^%w4d)#g}4#c_+1}=Oi#A2nm%)_wx*cgW{2xf z!&L?*yts;X|ClS~RN;@}5_bR5B&iD(aj0gcgaHPXmJU%cgzyBXVH9b*?S#L#KPf1S zl?bl$T$`hC@Ts}KfEk@8!wHI#7hUdlerM5@@)D4pfbp1hkS1#J$ybkzTaRs)fuo{# ztUM*E6RcU>S6&V0nCHP;y zwUnLgEu4&Op^JA{d-J=3avRgdpM(~!9`7J6J&PM(vW&+X8ieii5^U|TW61yX&WxKy zk7CMQ*QG~e#i=|Qoz7mjq}^N)jL85bU>=$$1MqPEjjHJDYOJG0#n ztZa@IveWt~^%*`E9?>Q>OSj#Ex15L6*iE=z#o|deaAu@(x_Mf=w=l0@eJY<#3v@K5 z#@*|IlrJ`skS!1|GGbmIn7vy*wbt4Pz96za>FGZ6#Y0*SUHpVUk`GwBHMV-ksW+`h zLfCq~F3s6T7pFU@PoiT+5W;{nVMgY5-$<1Am1SgV%%{Z@{fPMHMNx@KVwbfQ`ktXl z#m=t4f%QhO zf9hGzGNQs}b{#Xnvc0!BUaZ+BHJTp_^+Wudc2uZIgyBAUdTD{5)2(ig8w>8sC>1C? zPU$vE<AQDjGF)(gyd+I@qX5KUpq6v}!Pw;#bzdTmU9)Q4i{~PD`fp;bDpa3#`rF z)S7N{_U^o>bz?PY$SLk2M5{IRVb`D18vGQ;e=j`ca|0)D^V|d*Hw)jRGT#9LG%hyU z<}lBIbH#>2idEvPsN!?M-pyQ*yho9fRNkcS87nWl3Tov^&@4MN?x>2+;N7zCZTD(YwC)vcb_c0gbW|i+%~G(-fv)|lL+1lf1n!ylz}`GH ziQ}k{atK=KE9GV%q%~{!Nl!2kO~s-oj#Tl(A^EnkemO@@c#R*O`Owaa>p93|XK(8M zU<2bmBs9}0LAI?=X0f<0dAR+&Pum~}Z$;6NV-)iK*(Dnm#%j0D^SmB}fU0>kq@qHM zht#%xMCKE;8k)_gZ-<>w8FccGF>AY;eegoLmmgK&6J)bg#rrW_Hdhs;FPksU`*m}M zL3@8bYPL&m^o9-w&lRd`3QlKXg(n|uKw)u8Ug5>qG1;09sN*^D9-;-UmP=iZ%>;SLtjUojvskoL4GMT(yipZ!BzG9@A;{Eb%k3_bRhj_Nm_UV~x4&m=^lj zafM_0zZ#+7S5 zGjp59rtheO$R?_uKz>9~nJ~AUDJQcx&Yrb{K*>9;$AUUQy!JxV_pNVkTJ$Ua8w%<^W3d2R54@-TQ5eI?M^fM(m~6 zcMw|YmZsut3k5k)kC676$*+zE4otZ#1oAY+zhQ;K&CZbySFv=CZBczZIDfanUd816 zRVsX>gy!3|Y=4~miSn}1(o;F35!t5|@-}YKWR7d>G8zv=_b!8Y&4D9_ZVXg%1FwfU za}*h6Lz43otKMI#;mNAN#V1L0q}P1qAljo}VdNn`)H!HZd|wl&snkqudGLH4f>T$d z&xJ$bs_nsL{>JOg3s6E#3&ptbD0Yp>kz6Bcnw6@SORRdw`kFio)}U3hkxn8BQ`u<-r5Mv-=xBnz600T))I)?v)%)f7yKA$3A`=|*3Yds-@%kzj2*M(<5iCDJr60S!JM(#Bg@ooQm7?xVzn`PeVS(K$r8IhV`qT>X zv#jw$f~)iDmG%Sr*2Nb}49!P_xh!sGA72O}E))#(r5eBC-q->+^vu_eh4uAEOBD7v zJC%MC%Q9Hu+Erq>M{`w%UcNZ+c%DmHm0|@shWN-$$Pzg-gyb`TLJa;vU*QL-XYWed zu#Iv>J9`k-eN&vfDXYmxL(KR*EB2|R^t@U9YJiOs!q-=bzk0>0KvDU5406#uXpI_-0_y#|2?M0HWuy~>z*);Dqvqi`huWLv=NvL`5MAyp%ddq?! zICD5J+&^8F_b}26dHGFBgIZ4eh@6apjoU)&!wk98mC8)W{j3~SR5Z!W5d zxPwco9pT2sg=s{7ktpfjjiZ6H8)|}* zA$9G5G(`6uH=OS7v{4@1ot5;g86TUY$l>YsdE`dHTg$UeACw z8!YA_lE7d2_-PToy^#Hm{T0A}lem5c@I4=pKVkm+(!!&Q`wNhY`@8*5;O~MHEKnQx zYsP+y>z5b<8(|Lb571BWPy3(1-}N3?pe*z(`Dcp$0EAV4VBypXKj1$f;-`g$|63D* zh38`YKj1$U5?HW6ednYUG>@#YKkT&q;-T<=$?NYj1}qqi`3ErU6#Ok}z>+mme}p^` z`n!b<`QN*5STYOtAILC!aaf`d>+frLvg`*LmiX@tFD#K6=MThR+h16qsLKyPzv1r| z7WnV)eOO=u^pNAfb^)*$z}Jm(@3e8a$Cu%?->n->*WV=o5efQ_2f%{<2tfa;yZ8O> Fe*mg-?{feE diff --git a/iotedgedev/utility.py b/iotedgedev/utility.py index bca6be5..13fc09b 100644 --- a/iotedgedev/utility.py +++ b/iotedgedev/utility.py @@ -1,17 +1,19 @@ -from base64 import b64encode, b64decode import fnmatch -from hashlib import sha256 -from hmac import HMAC import json import os import subprocess import sys +from base64 import b64decode, b64encode +from hashlib import sha256 +from hmac import HMAC from time import time + +from .moduletype import ModuleType + if sys.version_info.major >= 3: from urllib.parse import quote, urlencode else: from urllib import quote, urlencode -from .moduletype import ModuleType class Utility: @@ -78,9 +80,13 @@ class Utility: return "SharedAccessSignature " + urlencode(rawtoken) - def get_file_contents(self, file): + def get_file_contents(self, file, expand_env=False): with open(file, "r") as file: - return file.read() + content = file.read() + if expand_env: + return os.path.expandvars(content) + else: + return content def decode(self, val): return val.decode("utf-8").strip() @@ -90,9 +96,12 @@ class Utility: return [os.path.join(os.getcwd(), f) for f in os.listdir(os.getcwd()) if f.endswith("template.json")] - def get_active_modules(self): + def get_bypass_modules(self): return [module.strip() - for module in self.envvars.ACTIVE_MODULES.split(",") if module] + for module in self.envvars.BYPASS_MODULES.split(",") if module] + + def get_active_docker_platform(self): + return [platform.strip() for platform in self.envvars.ACTIVE_DOCKER_PLATFORMS.split(",") if platform] def get_modules_in_config(self, moduleType): modules_config = json.load(open(self.envvars.DEPLOYMENT_CONFIG_FILE_PATH)) @@ -112,7 +121,7 @@ class Utility: return_modules.update(user_modules) return return_modules - def set_config(self, force=False): + def set_config(self, force=False, replacements=None): if not self.config_set or force: self.output.header("PROCESSING CONFIG FILES") @@ -137,12 +146,34 @@ class Utility: self.output.info("Expanding '{0}' to '{1}'".format( os.path.basename(config_file), build_config_file)) - config_file_expanded = os.path.expandvars( - self.get_file_contents(config_file)) - - with open(build_config_file, "w") as config_file_build: - config_file_build.write(config_file_expanded) + self.copy_template(config_file, build_config_file, replacements, True) self.output.line() self.config_set = True + + def copy_template(self, src, dest=None, replacements=None, expand_env=True): + """Read file at src, replace the keys in replacements with their values, optionally expand environment variables, and save to dest""" + if dest is None: + dest = src + + content = self.get_file_contents(src) + + if replacements: + for key, value in replacements.items(): + content = content.replace(key, value) + + if expand_env: + content = os.path.expandvars(content) + + with open(dest, "w") as dest_file: + dest_file.write(content) + + def nested_set(self, dic, keys, value): + current = dic + for key in keys[:-1]: + if key not in current: + current[key] = {} + current = current.get(key) + + current[keys[-1]] = value diff --git a/requirements.txt b/requirements.txt index f500aea..2211c91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ -docker==3.0.0 +docker>=3.4 python-dotenv requests fstrings -azure-iot-edge-runtime-ctl==1.0.0rc22 azure-cli-iot azure-cli-profile azure-cli-extension diff --git a/setup.cfg b/setup.cfg index 8d4e954..d0a4c04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,5 @@ exclude = docs [aliases] +[pytest] +norecursedirs=tests/utility diff --git a/setup.py b/setup.py index 6128ec0..698fcac 100644 --- a/setup.py +++ b/setup.py @@ -31,11 +31,10 @@ with open('HISTORY.rst') as history_file: requirements = [ 'Click>=6.0', - 'docker==3.0.0', + 'docker>=3.4', 'python-dotenv', 'requests', 'fstrings', - 'azure-iot-edge-runtime-ctl==1.0.0rc22', 'azure-cli-iot', 'azure-cli-profile', 'azure-cli-extension', diff --git a/tests/assets/deployment.template_1.json b/tests/assets/deployment.template_1.json new file mode 100644 index 0000000..fbad700 --- /dev/null +++ b/tests/assets/deployment.template_1.json @@ -0,0 +1,79 @@ +{ + "moduleContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.0", + "runtime": { + "type": "docker", + "settings": { + "minDockerVersion": "v1.25", + "loggingOptions": "" + } + }, + "systemModules": { + "edgeAgent": { + "type": "docker", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.0", + "createOptions": "" + } + }, + "edgeHub": { + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.0", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" + } + } + }, + "modules": { + "temp-sensor-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0", + "createOptions": "" + } + }, + "csharpmodule": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpmodule.amd64}", + "createOptions": "" + } + }, + "csharpfunction": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpfunction.amd64.debug}", + "createOptions": "" + } + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.0", + "routes": { + "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")", + "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream", + "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream" + }, + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/tests/assets/deployment.template_2.json b/tests/assets/deployment.template_2.json new file mode 100644 index 0000000..a837d4e --- /dev/null +++ b/tests/assets/deployment.template_2.json @@ -0,0 +1,79 @@ +{ + "moduleContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.0", + "runtime": { + "type": "docker", + "settings": { + "minDockerVersion": "v1.25", + "loggingOptions": "" + } + }, + "systemModules": { + "edgeAgent": { + "type": "docker", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.0", + "createOptions": "" + } + }, + "edgeHub": { + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.0", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" + } + } + }, + "modules": { + "temp-sensor-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0", + "createOptions": "" + } + }, + "csharpmodule": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "localhost:5000/csharpmodule:0.0.1-amd64", + "createOptions": "" + } + }, + "csharpfunction": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "localhost:5000/csharpfunction:0.0.1-amd64.debug", + "createOptions": "" + } + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.0", + "routes": { + "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")", + "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream", + "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream" + }, + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/tests/assets/deployment.template_3.json b/tests/assets/deployment.template_3.json new file mode 100644 index 0000000..d654b84 --- /dev/null +++ b/tests/assets/deployment.template_3.json @@ -0,0 +1,90 @@ +{ + "moduleContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.0", + "runtime": { + "type": "docker", + "settings": { + "minDockerVersion": "v1.25", + "loggingOptions": "" + } + }, + "systemModules": { + "edgeAgent": { + "type": "docker", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.0", + "createOptions": "" + } + }, + "edgeHub": { + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.0", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" + } + } + }, + "modules": { + "temp-sensor-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0", + "createOptions": "" + } + }, + "csharpmodule": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpmodule.amd64}", + "createOptions": "" + } + }, + "csharpfunction": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpfunction.amd64.debug}", + "createOptions": "" + } + }, + "csharpmodule2": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpmodule2.amd64}", + "createOptions": "" + } + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.0", + "routes": { + "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")", + "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream", + "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream", + "csharpmodule2ToIoTHub": "FROM /messages/modules/csharpmodule2/outputs/* INTO $upstream" + }, + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8f17ebe --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "utility")) diff --git a/tests/node_solution/.env.tmp b/tests/node_solution/.env.tmp index a559f85..4eeccab 100644 --- a/tests/node_solution/.env.tmp +++ b/tests/node_solution/.env.tmp @@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="." RUNTIME_HOST_NAME="." # "." - Auto detect -RUNTIME_TAG="1.0-preview" +RUNTIME_TAG="1.0" RUNTIME_VERBOSITY="INFO" # "DEBUG", "INFO", "ERROR", "WARNING" diff --git a/tests/test_azurecli.py b/tests/test_azurecli.py index d5f2e06..0fe44c7 100644 --- a/tests/test_azurecli.py +++ b/tests/test_azurecli.py @@ -1,29 +1,33 @@ import pytest + from iotedgedev.azurecli import get_query_argument_for_id_and_name pytestmark = pytest.mark.unit + def get_terms(query): - # These tests are all asserting that the query contains two terms enclosed in - # [?], separated by || - # They don't care about the order. Tests will fail if the square brackets and || + # These tests are all asserting that the query contains two terms enclosed in + # [?], separated by || + # They don't care about the order. Tests will fail if the square brackets and || # contract is violated, but we'd want them to fail in that case. return query[2:len(query)-1].split(" || ") + def test_lowercase_token_should_be_lowercase_for_name_and_id(): token = "abc123" query = get_query_argument_for_id_and_name(token) terms = get_terms(query) assert len(terms) == 2 - assert "starts_with(@.id,'abc123')" in terms + assert "starts_with(@.id,'abc123')" in terms assert "contains(@.name,'abc123')" in terms + def test_mixedcase_token_should_be_lowercase_for_id_but_unmodified_for_name(): token = "AbC123" query = get_query_argument_for_id_and_name(token) terms = get_terms(query) assert len(terms) == 2 - assert "starts_with(@.id,'abc123')" in terms - assert "contains(@.name,'AbC123')" in terms \ No newline at end of file + assert "starts_with(@.id,'abc123')" in terms + assert "contains(@.name,'AbC123')" in terms diff --git a/tests/test_connectionstring.py b/tests/test_connectionstring.py index 509daa0..19a2ba6 100644 --- a/tests/test_connectionstring.py +++ b/tests/test_connectionstring.py @@ -1,7 +1,8 @@ -import os import pytest -from dotenv import load_dotenv -from iotedgedev.connectionstring import ConnectionString, IoTHubConnectionString, DeviceConnectionString + +from iotedgedev.connectionstring import (ConnectionString, + DeviceConnectionString, + IoTHubConnectionString) pytestmark = pytest.mark.unit @@ -13,24 +14,29 @@ invalid_connectionstring = "HostName=azure-devices.net;SharedAccessKey=gibberish invalid_iothub_connectionstring = "HostName=testhub.azure-devices.net;SharedAccessKey=moregibberish" invalid_device_connectionstring = "HostName=testhub.azure-devices.net;DeviceId=;SharedAccessKey=othergibberish" + def test_empty_connectionstring(): connectionstring = ConnectionString(emptystring) assert not connectionstring.data + def test_empty_iothub_connectionstring(): connectionstring = IoTHubConnectionString(emptystring) assert not connectionstring.data + def test_empty_device_connectionstring(): connectionstring = DeviceConnectionString(emptystring) assert not connectionstring.data + def test_valid_connectionstring(): connectionstring = ConnectionString(valid_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" assert connectionstring.HubName == "testhub" assert connectionstring.SharedAccessKey == "gibberish" + def test_valid_iothub_connectionstring(): connectionstring = IoTHubConnectionString(valid_iothub_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" @@ -38,6 +44,7 @@ def test_valid_iothub_connectionstring(): assert connectionstring.SharedAccessKeyName == "iothubowner" assert connectionstring.SharedAccessKey == "moregibberish" + def test_valid_devicehub_connectionstring(): connectionstring = DeviceConnectionString(valid_device_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" @@ -45,17 +52,20 @@ def test_valid_devicehub_connectionstring(): assert connectionstring.DeviceId == "testdevice" assert connectionstring.SharedAccessKey == "othergibberish" + def test_invalid_connectionstring(): connectionstring = ConnectionString(invalid_connectionstring) assert connectionstring.HubName != "testhub" + def test_invalid_iothub_connectionstring(): with pytest.raises(KeyError): IoTHubConnectionString(invalid_iothub_connectionstring) + def test_invalid_devicehub_connectionstring(): connectionstring = DeviceConnectionString(invalid_device_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" assert connectionstring.HubName == "testhub" assert not connectionstring.DeviceId - assert connectionstring.SharedAccessKey == "othergibberish" \ No newline at end of file + assert connectionstring.SharedAccessKey == "othergibberish" diff --git a/tests/test_deploymentmanifest.py b/tests/test_deploymentmanifest.py new file mode 100644 index 0000000..3a2921a --- /dev/null +++ b/tests/test_deploymentmanifest.py @@ -0,0 +1,50 @@ +import json +import os + +import pytest + +from iotedgedev.deploymentmanifest import DeploymentManifest +from iotedgedev.envvars import EnvVars +from iotedgedev.output import Output +from iotedgedev.utility import Utility +from utility import assert_list_equal + +pytestmark = pytest.mark.unit + +tests_dir = os.path.join(os.getcwd(), "tests") +test_assets_dir = os.path.join(tests_dir, "assets") +test_file_1 = os.path.join(test_assets_dir, "deployment.template_1.json") +test_file_2 = os.path.join(test_assets_dir, "deployment.template_2.json") +test_file_3 = os.path.join(test_assets_dir, "deployment.template_3.json") + + +@pytest.fixture +def deployment_manifest(): + output = Output() + envvars = EnvVars(output) + envvars.load() + utility = Utility(envvars, output) + + def _deployment_manifest(path): + return DeploymentManifest(envvars, output, utility, path, True) + + return _deployment_manifest + + +def test_get_modules_to_process(deployment_manifest): + deployment_manifest = deployment_manifest(test_file_1) + modules_to_process = deployment_manifest.get_modules_to_process() + assert_list_equal(modules_to_process, [("csharpmodule", "amd64"), ("csharpfunction", "amd64.debug")]) + + +def test_get_modules_to_process_empty(deployment_manifest): + deployment_manifest = deployment_manifest(test_file_2) + modules_to_process = deployment_manifest.get_modules_to_process() + assert_list_equal(modules_to_process, []) + + +def test_add_module_template(deployment_manifest): + deployment_manifest = deployment_manifest(test_file_1) + deployment_manifest.add_module_template("csharpmodule2") + with open(test_file_3, "r") as expected: + assert deployment_manifest.json == json.load(expected) diff --git a/tests/test_envvars.py b/tests/test_envvars.py index daf0bc3..340810a 100644 --- a/tests/test_envvars.py +++ b/tests/test_envvars.py @@ -5,24 +5,28 @@ from iotedgedev.output import Output pytestmark = pytest.mark.unit + def test_valid_get_envvar(): output = Output() envvars = EnvVars(output) loglevel = envvars.get_envvar("RUNTIME_LOG_LEVEL") assert loglevel == "info" or "debug" + def test_invalid_get_envvar(): output = Output() envvars = EnvVars(output) testerval = envvars.get_envvar("TESTER") assert not testerval - + + def test_valid_load(): output = Output() envvars = EnvVars(output) envvars.load() assert envvars.RUNTIME_LOG_LEVEL == "info" or "debug" + def test_valid_verify_envvar_has_val(): output = Output() envvars = EnvVars(output) @@ -30,16 +34,19 @@ def test_valid_verify_envvar_has_val(): result = envvars.verify_envvar_has_val("RUNTIME_LOG_LEVEL", envvars.RUNTIME_LOG_LEVEL) assert not result + def test_valid_get_envvar_key_if_val(): output = Output() envvars = EnvVars(output) assert envvars.get_envvar_key_if_val("RUNTIME_LOG_LEVEL") + def test_invalid_get_envvar_key_if_val(): output = Output() envvars = EnvVars(output) assert not envvars.get_envvar_key_if_val("TESTER") + def test_set_envvar(): output = Output() envvars = EnvVars(output) @@ -48,4 +55,3 @@ def test_set_envvar(): setlevel = envvars.get_envvar("RUNTIME_LOG_LEVEL") assert setlevel == "debug" envvars.set_envvar("RUNTIME_LOG_LEVEL", loglevel) - \ No newline at end of file diff --git a/tests/test_iotedgedev.py b/tests/test_iotedgedev.py index 9b50ee7..0c1438c 100644 --- a/tests/test_iotedgedev.py +++ b/tests/test_iotedgedev.py @@ -1,9 +1,13 @@ import json import os import shutil -import pytest +import pytest from click.testing import CliRunner +from dotenv import load_dotenv + +from iotedgedev.connectionstring import (DeviceConnectionString, + IoTHubConnectionString) pytestmark = pytest.mark.e2e @@ -106,7 +110,7 @@ def test_module_add(): runner = CliRunner() add_module_and_verify(cli.main, runner, "csharp") - # add_module_and_verify(cli.main, runner, "nodejs") + add_module_and_verify(cli.main, runner, "nodejs") add_module_and_verify(cli.main, runner, "python") add_module_and_verify(cli.main, runner, "csharpfunction") @@ -161,49 +165,49 @@ def test_deploy_modules(request): assert 'DEPLOYMENT COMPLETE' in result.output -@pytest.fixture -def test_start_runtime(request): +# @pytest.fixture +# def test_start_runtime(request): - os.chdir(test_solution_dir) +# os.chdir(test_solution_dir) - cli = __import__("iotedgedev.cli", fromlist=['main']) - runner = CliRunner() - result = runner.invoke(cli.main, ['start']) - print(result.output) +# cli = __import__("iotedgedev.cli", fromlist=['main']) +# runner = CliRunner() +# result = runner.invoke(cli.main, ['start']) +# print(result.output) - assert 'Runtime started' in result.output +# assert 'Runtime started' in result.output -@pytest.fixture -def test_monitor(request, capfd): +# @pytest.fixture +# def test_monitor(request, capfd): - os.chdir(test_solution_dir) +# os.chdir(test_solution_dir) - cli = __import__("iotedgedev.cli", fromlist=['main']) - runner = CliRunner() - result = runner.invoke(cli.main, ['monitor', '--timeout', '60000']) - out, err = capfd.readouterr() - print(out) - print(err) - print(result.output) +# cli = __import__("iotedgedev.cli", fromlist=['main']) +# runner = CliRunner() +# result = runner.invoke(cli.main, ['monitor', '--timeout', '60000']) +# out, err = capfd.readouterr() +# print(out) +# print(err) +# print(result.output) - assert 'timeCreated' in out +# assert 'timeCreated' in out -@pytest.fixture -def test_stop(request): +# @pytest.fixture +# def test_stop(request): - os.chdir(test_solution_dir) +# os.chdir(test_solution_dir) - cli = __import__("iotedgedev.cli", fromlist=['main']) - runner = CliRunner() - result = runner.invoke(cli.main, ['stop']) - print(result.output) +# cli = __import__("iotedgedev.cli", fromlist=['main']) +# runner = CliRunner() +# result = runner.invoke(cli.main, ['stop']) +# print(result.output) - assert 'Runtime stopped' in result.output +# assert 'Runtime stopped' in result.output -def test_e2e(test_push_modules, test_deploy_modules, test_start_runtime, test_monitor, test_stop): +def test_e2e(test_push_modules, test_deploy_modules): print('Testing E2E') @@ -219,9 +223,10 @@ def setup_node_solution(request): return -def test_node(setup_node_solution, test_push_modules, test_deploy_modules, test_start_runtime, test_monitor, test_stop): +def test_node(setup_node_solution, test_push_modules, test_deploy_modules): print('Testing Node Solution') + def test_valid_env_iothub_connectionstring(): load_dotenv(".env") env_iothub_connectionstring = os.getenv("IOTHUB_CONNECTION_STRING") @@ -231,6 +236,7 @@ def test_valid_env_iothub_connectionstring(): assert connectionstring.SharedAccessKey assert connectionstring.SharedAccessKeyName + def test_valid_env_device_connectionstring(): load_dotenv(".env") env_device_connectionstring = os.getenv("DEVICE_CONNECTION_STRING") @@ -240,6 +246,7 @@ def test_valid_env_device_connectionstring(): assert connectionstring.SharedAccessKey assert connectionstring.DeviceId + ''' def test_load_no_dotenv(): diff --git a/tests/test_utility.py b/tests/test_utility.py new file mode 100644 index 0000000..6786256 --- /dev/null +++ b/tests/test_utility.py @@ -0,0 +1,44 @@ +import os + +import pytest + +from iotedgedev.envvars import EnvVars +from iotedgedev.output import Output +from iotedgedev.utility import Utility +from utility import assert_json_file_equal + +pytestmark = pytest.mark.unit + +tests_dir = os.path.join(os.getcwd(), "tests") +test_assets_dir = os.path.join(tests_dir, "assets") +test_file_1 = os.path.join(test_assets_dir, "deployment.template_1.json") +test_file_2 = os.path.join(test_assets_dir, "deployment.template_2.json") + + +@pytest.fixture +def utility(): + output = Output() + envvars = EnvVars(output) + envvars.load() + return Utility(envvars, output) + + +def test_copy_template(utility, tmpdir): + replacements = { + "${MODULES.csharpmodule.amd64}": "localhost:5000/csharpmodule:0.0.1-amd64", + "${MODULES.csharpfunction.amd64.debug}": "localhost:5000/csharpfunction:0.0.1-amd64.debug" + } + dest = tmpdir.join("deployment_template_1.dest.json").strpath + utility.copy_template(test_file_1, dest, replacements=replacements, expand_env=False) + assert_json_file_equal(test_file_2, dest) + + +def test_copy_template_expand_env(utility, tmpdir): + replacements = { + "${MODULES.csharpmodule.amd64}": "${CONTAINER_REGISTRY_SERVER}/csharpmodule:0.0.1-amd64", + "${MODULES.csharpfunction.amd64.debug}": "${CONTAINER_REGISTRY_SERVER}/csharpfunction:0.0.1-amd64.debug" + } + os.environ["CONTAINER_REGISTRY_SERVER"] = "localhost:5000" + dest = tmpdir.join("deployment_template_2.dest.json").strpath + utility.copy_template(test_file_1, dest, replacements=replacements, expand_env=True) + assert_json_file_equal(test_file_2, dest) diff --git a/tests/utility/utility.py b/tests/utility/utility.py new file mode 100644 index 0000000..36999b5 --- /dev/null +++ b/tests/utility/utility.py @@ -0,0 +1,11 @@ +import json + + +def assert_list_equal(list1, list2): + return len(list1) == len(list2) and sorted(list1) == sorted(list2) + + +def assert_json_file_equal(file1, file2): + with open(file1, "r") as f1: + with open(file2, "r") as f2: + assert json.load(f1) == json.load(f2)