Add --template param to solution creation (#194)

* Use  env key instead of value when adding modules

* Add CLI entrypoint

* Call iotedgedev addmodule when create solution

* Add default routes when adding modules

* Update template

* Update test

* Revert changes to monitor timeout

* Add temp sensor route when adding module

* Add copy_template function

* Add default ToIoTHub route when creating solution

* Update system module images to GA version

* Fix incorrect string escape

* Replace image placeholder with real image URL when building images

* Enable Node.js module creation

* Refind the logic to parse image placeholder

* Won't add tempsensor route when adding modules

* Minor refinement

* Add nested_set utility method

* Add default route from temp sensor when solution creation

* Rename var_dict to replacement

* Use name in env as default when new solution

* WIP support for BYPASS_MODULES

* WIP support for BYPASS_MODULES

* WIP support for BYPASS_MODULES

* Update utility methods

* Update image_tag_map key type to tuple

* Update Docker SDK version and remove iotedgeruntime from requirements.txt

* Disable outdated test cases temporarily

* Add unit test for deploymentmanifest.py

* Add unit test for utility.py

* Fix error on Py27

* Compare lists order-insensitively

* Fix PyTest failure on Python 3

.env.tmp

@@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="."
 RUNTIME_HOST_NAME="."
     # "." - Auto detect
 
-RUNTIME_TAG="1.0-preview"
+RUNTIME_TAG="1.0"
 
 RUNTIME_VERBOSITY="INFO"
     # "DEBUG", "INFO", "ERROR", "WARNING"
@@ -51,9 +51,9 @@ RUNTIME_LOG_LEVEL="info"
 # MODULES
 #
 
-ACTIVE_MODULES="*" 
-    # "*" - to build all modules
-    # "filtermodule, module1" - Comma delimited list of modules to build
+BYPASS_MODULES=""
+    # "" - to build all modules
+    # "filtermodule, module1" - Comma delimited list of modules to bypass when building
 
 ACTIVE_DOCKER_PLATFORMS="amd64"
     # "*" - to build all docker files

iotedgedev/cli.py

@@ -61,14 +61,25 @@ def main(set_config, az_cli=None):
               required=False,
               help="Creates a new Azure IoT Edge Solution. Use `--create .` to create in current folder. Use `--create TEXT` to create in a subfolder.")
 @click.argument("name", required=False)
-def solution(create, name):
+@click.option('--module',
+              required=False,
+              default=envvars.get_envvar("DEFAULT_MODULE_NAME", default="filtermodule"),
+              show_default=True,
+              help="Specify the name of the default IoT Edge module.")
+@click.option("--template",
+              default="csharp",
+              show_default=True,
+              required=False,
+              type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]),
+              help="Specify the template used to create the default IoT Edge module.")
+def solution(create, name, module, template):
 
     utility = Utility(envvars, output)
     sol = Solution(output, utility)
     if name:
-        sol.create(name)
+        sol.create(name, module, template)
     elif create:
-        sol.create(create)
+        sol.create(create, module, template)
 
 
 @click.command(context_settings=CONTEXT_SETTINGS, help="Creates Solution and Azure Resources")
@@ -107,7 +118,7 @@ def e2e(ctx):
                 required=True)
 @click.option("--template",
               required=True,
-              type=click.Choice(["csharp", "python", "csharpfunction"]),
+              type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]),
               help="Specify the template used to create the new IoT Edge module.")
 @click.pass_context
 def addmodule(ctx, name, template):
@@ -444,7 +455,7 @@ def azure(setup,
 @click.option("--template",
               default="csharp",
               required=False,
-              type=click.Choice(["csharp", "python", "csharpfunction"]),
+              type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]),
               help="Specify the template used to create the new IoT Edge module.")
 @click.option('--build',
               default=False,

iotedgedev/deploymentmanifest.py

@@ -10,12 +10,13 @@ import sys
 
 
 class DeploymentManifest:
-    def __init__(self, envvars, output, path, is_template):
+    def __init__(self, envvars, output, utility, path, is_template):
+        self.utility = utility
         self.output = output
         try:
             self.path = path
             self.is_template = is_template
-            self.json = json.load(open(path))
+            self.json = json.loads(self.utility.get_file_contents(path, expand_env=True))
         except FileNotFoundError:
             self.output.error('Deployment manifest template file "{0}" not found'.format(path))
             if is_template:
@@ -27,7 +28,7 @@ class DeploymentManifest:
                         envvars.save_envvar("DEPLOYMENT_CONFIG_TEMPLATE_FILE", path)
             else:
                 self.output.error('Deployment manifest file "{0}" not found'.format(path))
-                sys.exit()
+                sys.exit(1)
 
     def add_module_template(self, module_name):
         """Add a module template to the deployment manifest with amd64 as the default platform"""
@@ -37,12 +38,36 @@ class DeploymentManifest:
             "status": "running",
             "restartPolicy": "always",
             "settings": {
-              "image": \"{MODULES.""" + module_name + """.amd64}\",
+              "image": \"${MODULES.""" + module_name + """.amd64}\",
               "createOptions": ""
             }
         }"""
 
-        self.json["moduleContent"]["$edgeAgent"]["properties.desired"]["modules"][module_name] = json.loads(new_module)
+        self.utility.nested_set(self.json, ["moduleContent", "$edgeAgent", "properties.desired", "modules", module_name], json.loads(new_module))
+
+        self.add_default_route(module_name)
+
+    def add_default_route(self, module_name):
+        """Add a default route to send messages to IoT Hub"""
+        new_route_name = "{0}ToIoTHub".format(module_name)
+        new_route = "FROM /messages/modules/{0}/outputs/* INTO $upstream".format(module_name)
+
+        self.utility.nested_set(self.json, ["moduleContent", "$edgeHub", "properties.desired", "routes", new_route_name], new_route)
+
+    def get_modules_to_process(self):
+        """Get modules to process from deployment manifest template"""
+        user_modules = self.json.get("moduleContent", {}).get("$edgeAgent", {}).get("properties.desired", {}).get("modules", {})
+        modules_to_process = []
+        for _, module_info in user_modules.items():
+            image = module_info.get("settings", {}).get("image", "")
+            # If the image is placeholder, e.g., ${MODULES.NodeModule.amd64}, parse module folder and platform from the placeholder
+            if image.startswith("${") and image.endswith("}") and len(image.split(".")) > 2:
+                first_dot = image.index(".")
+                second_dot = image.index(".", first_dot + 1)
+                module_dir = image[first_dot+1:second_dot]
+                module_platform = image[second_dot+1:image.index("}")]
+                modules_to_process.append((module_dir, module_platform))
+        return modules_to_process
 
     def save(self):
         """Dump the JSON to the disk"""

iotedgedev/dockercls.py

@@ -23,6 +23,9 @@ class Docker:
             self.docker_client = docker.from_env()
             self.docker_api = docker.APIClient()
 
+    def get_os_type(self):
+        return self.docker_client.info()["OSType"].lower()
+
     def init_registry(self):
 
         self.output.header("INITIALIZING CONTAINER REGISTRY")

iotedgedev/envvars.py

@@ -109,7 +109,7 @@ class EnvVars:
                 self.RUNTIME_CONFIG_DIR = self.get_envvar("RUNTIME_CONFIG_DIR", default=".")
                 if self.RUNTIME_CONFIG_DIR == ".":
                     self.set_envvar("RUNTIME_CONFIG_DIR", self.get_runtime_config_dir())
-                self.ACTIVE_MODULES = self.get_envvar("ACTIVE_MODULES")
+                self.BYPASS_MODULES = self.get_envvar("BYPASS_MODULES")
                 self.ACTIVE_DOCKER_PLATFORMS = self.get_envvar("ACTIVE_DOCKER_PLATFORMS", altkeys=["ACTIVE_DOCKER_ARCH"])
                 self.CONTAINER_REGISTRY_SERVER = self.get_envvar("CONTAINER_REGISTRY_SERVER")
                 self.CONTAINER_REGISTRY_USERNAME = self.get_envvar("CONTAINER_REGISTRY_USERNAME")

iotedgedev/module.py

@@ -1,6 +1,5 @@
-
-import os
 import json
+import os
 import sys
 
 
@@ -17,8 +16,7 @@ class Module(object):
     def load_module_json(self):
         if os.path.exists(self.module_json_file):
             try:
-                self.file_json_content = json.loads(
-                    self.utility.get_file_contents(self.module_json_file))
+                self.file_json_content = json.loads(self.utility.get_file_contents(self.module_json_file, expand_env=True))
 
                 self.module_language = self.file_json_content.get(
                     "language").lower()
@@ -37,15 +35,21 @@ class Module(object):
 
     @property
     def platforms(self):
-        return self.file_json_content.get("image").get("tag").get("platforms")
+        return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", "")
 
     @property
     def tag_version(self):
-        tag = self.file_json_content.get("image").get("tag").get("version")
-        if tag == "":
-            tag = "0.0.0"
+        tag = self.file_json_content.get("image", {}).get("tag", {}).get("version", "0.0.0")
 
         return tag
 
-    def get_platform_by_key(self, platform):
-        return self.file_json_content.get("image").get("tag").get("platforms").get(platform)
+    @property
+    def repository(self):
+        return self.file_json_content.get("image", {}).get("repository", "")
+
+    @property
+    def build_options(self):
+        return self.file_json_content.get("image", {}).get("buildOptions", [])
+
+    def get_dockerfile_by_platform(self, platform):
+        return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", {}).get(platform, "")

iotedgedev/modules.py

@@ -1,17 +1,16 @@
 import os
 import re
+import sys
 
 from .deploymentmanifest import DeploymentManifest
 from .dotnet import DotNet
 from .module import Module
-from .modulesprocessorfactory import ModulesProcessorFactory
 
 
 class Modules:
     def __init__(self, envvars, utility, output, dock):
         self.envvars = envvars
         self.utility = utility
-        self.utility.set_config()
         self.output = output
         self.dock = dock
         self.dock.init_registry()
@@ -30,15 +29,15 @@ class Modules:
             self.output.error("Module \"{0}\" already exists under {1}".format(name, os.path.abspath(self.envvars.MODULES_PATH)))
             return
 
-        deployment_manifest = DeploymentManifest(self.envvars, self.output, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True)
+        deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True)
 
-        repo = "{0}/{1}".format(self.envvars.CONTAINER_REGISTRY_SERVER, name.lower())
+        repo = "{0}/{1}".format("${CONTAINER_REGISTRY_SERVER}", name.lower())
         if template == "csharp":
             dotnet = DotNet(self.envvars, self.output, self.utility)
             dotnet.install_module_template()
             dotnet.create_custom_module(name, repo, cwd)
         elif template == "nodejs":
-            self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator")
+            self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator", shell=True)
             cmd = "yo azure-iot-edge-module -n {0} -r {1}".format(name, repo)
             self.output.header(cmd)
             self.utility.exe_proc(cmd.split(), shell=True, cwd=cwd)
@@ -66,72 +65,97 @@ class Modules:
         self.build_push(no_build=no_build)
 
     def build_push(self, no_build=False, no_push=False):
-
         self.output.header("BUILDING MODULES", suppress=no_build)
 
-        # Get all the modules to build as specified in config.
-        modules_to_process = self.utility.get_active_modules()
+        bypass_modules = self.utility.get_bypass_modules()
+        active_platform = self.utility.get_active_docker_platform()
+
+        # map (module name, platform) tuple to tag.
+        # sample: (('filtermodule', 'amd64'), 'localhost:5000/filtermodule:0.0.1-amd64')
+        image_tag_map = {}
+        # map image tag to (module name, dockerfile) tuple
+        # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ('filtermodule', '/test_solution/modules/filtermodule/Dockerfile.amd64'))
+        tag_dockerfile_map = {}
+        # map image tag to build options
+        # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ["--add-host=github.com:192.30.255.112"])
+        tag_build_options_map = {}
+        # image tags to build
+        # sample: 'localhost:5000/filtermodule:0.0.1-amd64'
+        tags_to_build = set()
 
         for module in os.listdir(self.envvars.MODULES_PATH):
-
-            if len(modules_to_process) == 0 or modules_to_process[0] == "*" or module in modules_to_process:
-
+            if module not in bypass_modules:
                 module_dir = os.path.join(self.envvars.MODULES_PATH, module)
-
-                self.output.info("BUILDING MODULE: {0}".format(module_dir), suppress=no_build)
-
                 module_json = Module(self.output, self.utility, os.path.join(module_dir, "module.json"))
-                mod_proc = ModulesProcessorFactory(self.envvars, self.utility, self.output, module_dir).get(module_json.language)
+                for platform in module_json.platforms:
+                    # get the Dockerfile from module.json
+                    dockerfile = os.path.abspath(os.path.join(module_dir, module_json.get_dockerfile_by_platform(platform)))
+                    container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG
+                    tag = "{0}:{1}{2}-{3}".format(module_json.repository, module_json.tag_version, container_tag, platform).lower()
+                    image_tag_map[(module, platform)] = tag
+                    tag_dockerfile_map[tag] = (module, dockerfile)
+                    tag_build_options_map[tag] = module_json.build_options
+                    if len(active_platform) > 0 and (active_platform[0] == "*" or platform in active_platform):
+                        tags_to_build.add(tag)
 
-                # build module
+        deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True)
+        modules_to_process = deployment_manifest.get_modules_to_process()
+
+        replacements = {}
+        for module, platform in modules_to_process:
+            if module not in bypass_modules:
+                key = (module, platform)
+                if key in image_tag_map:
+                    tag = image_tag_map.get(key)
+                    tags_to_build.add(tag)
+                    replacements["${{MODULES.{0}.{1}}}".format(module, platform)] = tag
+
+        for tag in tags_to_build:
+            if tag in tag_dockerfile_map:
+                module = tag_dockerfile_map.get(tag)[0]
+                dockerfile = tag_dockerfile_map.get(tag)[1]
+                self.output.info("BUILDING MODULE: {0}".format(module), suppress=no_build)
+                self.output.info("PROCESSING DOCKERFILE: {0}".format(dockerfile), suppress=no_build)
+                self.output.info("BUILDING DOCKER IMAGE: {0}".format(tag), suppress=no_build)
+
+                # BUILD DOCKER IMAGE
                 if not no_build:
-                    if not mod_proc.build():
-                        continue
+                    # TODO: apply build options
+                    build_options = self.filter_build_options(tag_build_options_map.get(tag, None))
 
-                docker_arch_process = [docker_arch.strip() for docker_arch in self.envvars.ACTIVE_DOCKER_PLATFORMS.split(",") if docker_arch]
+                    context_path = os.path.abspath(os.path.join(self.envvars.MODULES_PATH, module))
+                    dockerfile_relative = os.path.relpath(dockerfile, context_path)
+                    # a hack to workaround Python Docker SDK's bug with Linux container mode on Windows
+                    if self.dock.get_os_type() == "linux" and sys.platform == "win32":
+                        dockerfile = dockerfile.replace("\\", "/")
+                        dockerfile_relative = dockerfile_relative.replace("\\", "/")
 
-                for arch in module_json.platforms:
-                    if len(docker_arch_process) == 0 or docker_arch_process[0] == "*" or arch in docker_arch_process:
+                    build_result = self.dock.docker_client.images.build(tag=tag, path=context_path, dockerfile=dockerfile_relative)
 
-                        # get the docker file from module.json
-                        docker_file = module_json.get_platform_by_key(arch)
+                    self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result))
 
-                        self.output.info("PROCESSING DOCKER FILE: " + docker_file, suppress=no_build)
+                if not no_push:
+                    # PUSH TO CONTAINER REGISTRY
+                    self.output.info("PUSHING DOCKER IMAGE: " + tag)
 
-                        docker_file_name = os.path.basename(docker_file)
-                        container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG
-                        tag_name = module_json.tag_version + container_tag
+                    for line in self.dock.docker_client.images.push(repository=tag, stream=True, auth_config={
+                            "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}):
+                        self.output.procout(self.utility.decode(line).replace("\\u003e", ">"))
+            self.output.footer("BUILD COMPLETE", suppress=no_build)
+            self.output.footer("PUSH COMPLETE", suppress=no_push)
+        self.utility.set_config(force=True, replacements=replacements)
 
-                        # publish module
-                        if not no_build:
-                            self.output.info("PUBLISHING MODULE: " + module_dir)
-                            mod_proc.publish()
+    @staticmethod
+    def filter_build_options(build_options):
+        """Remove build options which will be ignored"""
+        if build_options is None:
+            return None
 
-                        image_destination_name = "{0}/{1}:{2}-{3}".format(self.envvars.CONTAINER_REGISTRY_SERVER, module, tag_name, arch).lower()
+        filtered_build_options = []
+        for build_option in build_options:
+            build_option = build_option.strip()
+            parsed_option = re.compile(r"\s+").split(build_option)
+            if parsed_option and ["--rm", "--tag", "-t", "--file", "-f"].index(parsed_option[0]) < 0:
+                filtered_build_options.append(build_option)
 
-                        self.output.info("BUILDING DOCKER IMAGE: " + image_destination_name, suppress=no_build)
-
-                        # cd to the module folder to build the docker image
-                        project_dir = os.getcwd()
-                        os.chdir(os.path.join(project_dir, module_dir))
-
-                        # BUILD DOCKER IMAGE
-
-                        if not no_build:
-                            build_result = self.dock.docker_client.images.build(tag=image_destination_name, path=".", dockerfile=docker_file_name, buildargs={"EXE_DIR": mod_proc.exe_dir})
-
-                            self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result))
-
-                        # CD BACK UP
-                        os.chdir(project_dir)
-
-                        if not no_push:
-                            # PUSH TO CONTAINER REGISTRY
-                            self.output.info("PUSHING DOCKER IMAGE TO: " + image_destination_name)
-
-                            for line in self.dock.docker_client.images.push(repository=image_destination_name, stream=True, auth_config={
-                                                                            "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}):
-                                self.output.procout(self.utility.decode(line).replace("\\u003e", ">"))
-
-                self.output.footer("BUILD COMPLETE", suppress=no_build)
-                self.output.footer("PUSH COMPLETE", suppress=no_push)
+        return filtered_build_options

iotedgedev/runtime.py

@@ -32,4 +32,3 @@ class Runtime:
         self.dock.remove_modules()
         self.setup()
         self.start()
-

iotedgedev/solution.py

@@ -1,12 +1,13 @@
 import os
 import zipfile
 
+
 class Solution:
     def __init__(self, output, utility):
         self.output = output
         self.utility = utility
 
-    def create(self, name):
+    def create(self, name, module, template):
         if name == ".":
             dir_path = os.getcwd()
         else:
@@ -25,13 +26,17 @@ class Solution:
             self.output.error("Error while trying to load template.zip")
             self.output.error(str(ex))
 
-        if name == ".":
-            name = ""
-
         zipf = zipfile.ZipFile(template_zip)
         zipf.extractall(name)
 
+        self.utility.copy_template(os.path.join(dir_path, "deployment.template.json"), None, {"%MODULE%": module}, False)
+
         os.rename(os.path.join(name, ".env.tmp"), os.path.join(name, ".env"))
+
+        mod_cmd = "iotedgedev addmodule {0} --template {1}".format(module, template)
+        self.output.header(mod_cmd)
+        self.utility.call_proc(mod_cmd.split(), cwd=name)
+
         self.output.footer("Azure IoT Edge Solution Created")
-        if name != "":
+        if name != ".":
             self.output.info("Execute 'cd {0}' to navigate to your new solution.".format(name))

iotedgedev/template/.env.tmp

@@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="."
 RUNTIME_HOST_NAME="."
     # "." - Auto detect
 
-RUNTIME_TAG="1.0-preview"
+RUNTIME_TAG="1.0"
 
 RUNTIME_VERBOSITY="INFO"
     # "DEBUG", "INFO", "ERROR", "WARNING"
@@ -51,13 +51,14 @@ RUNTIME_LOG_LEVEL="info"
 # MODULES
 #
 
-ACTIVE_MODULES="*" 
-    # "*" - to build all modules
-    # "filtermodule, module1" - Comma delimited list of modules to build
+BYPASS_MODULES=""
+    # "" - to build all modules
+    # "filtermodule, module1" - Comma delimited list of modules to bypass when building
 
-ACTIVE_DOCKER_PLATFORMS="amd64"
-    # "*" - to build all docker files
-    # "amd64,amd64.debug" - Comma delimted list of docker files to build
+ACTIVE_DOCKER_PLATFORMS=""
+    # "" - to only build Dockerfiles specified in DEPLOYMENT_CONFIG_TEMPLATE_FILE
+    # "*" - to build all Dockerfiles
+    # "amd64,amd64.debug" - Comma delimited list of Dockerfiles to build
 
 CONTAINER_TAG="" 
 

iotedgedev/template/deployment.template.json

@@ -14,7 +14,7 @@
           "edgeAgent": {
             "type": "docker",
             "settings": {
-              "image": "microsoft/azureiotedge-agent:${RUNTIME_TAG}",
+              "image": "mcr.microsoft.com/azureiotedge-agent:${RUNTIME_TAG}",
               "createOptions": ""
             }
           },
@@ -23,8 +23,8 @@
             "status": "running",
             "restartPolicy": "always",
             "settings": {
-              "image": "microsoft/azureiotedge-hub:${RUNTIME_TAG}",
-              "createOptions": ""
+              "image": "mcr.microsoft.com/azureiotedge-hub:${RUNTIME_TAG}",
+              "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
             }
           }
         },
@@ -35,17 +35,7 @@
             "status": "running",
             "restartPolicy": "always",
             "settings": {
-              "image": "microsoft/azureiotedge-simulated-temperature-sensor:${RUNTIME_TAG}",
-              "createOptions": ""
-            }
-          },
-          "filtermodule": {
-            "version": "1.0",
-            "type": "docker",
-            "status": "running",
-            "restartPolicy": "always",
-            "settings": {
-              "image": "${CONTAINER_REGISTRY_SERVER}/filtermodule:0.0.1-amd64",
+              "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:${RUNTIME_TAG}",
               "createOptions": ""
             }
           }
@@ -56,19 +46,12 @@
       "properties.desired": {
         "schemaVersion": "1.0",
         "routes": {
-          "sensorToFilter": "FROM /messages/modules/temp-sensor-module/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/filtermodule/inputs/input1\")",
-          "filterToIoTHub": "FROM /messages/modules/filtermodule/outputs/output1 INTO $upstream"
+          "sensorTo%MODULE%": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/%MODULE%/inputs/input1\")"
         },
         "storeAndForwardConfiguration": {
           "timeToLiveSecs": 7200
         }
       }
-    },
-    "filtermodule": {
-      "properties.desired": {
-        "schemaVersion": "1.0",
-        "TemperatureThreshold": 21
-      }
     }
   }
 }

iotedgedev/template/modules/filtermodule/.gitignore

@@ -1,31 +0,0 @@
-# .NET Core
-project.lock.json
-project.fragment.lock.json
-artifacts/
-**/Properties/launchSettings.json
-
-*_i.c
-*_p.c
-*_i.h
-*.ilk
-*.meta
-*.obj
-*.pch
-*.pdb
-*.pgc
-*.pgd
-*.rsp
-*.sbr
-*.tlb
-*.tli
-*.tlh
-*.tmp
-*.tmp_proj
-*.log
-*.vspscc
-*.vssscc
-.builds
-*.pidb
-*.svclog
-*.scc
-.vs

iotedgedev/template/modules/filtermodule/Dockerfile

@@ -1,13 +0,0 @@
-FROM microsoft/dotnet:2.0-sdk AS build-env
-WORKDIR /app
-
-COPY *.csproj ./
-RUN dotnet restore
-
-COPY . ./
-RUN dotnet publish -c Release -o out
-
-FROM microsoft/dotnet:2.0-runtime
-WORKDIR /app
-COPY --from=build-env /app/out ./
-ENTRYPOINT ["dotnet", "filtermodule.dll"]

iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug

@@ -1,18 +0,0 @@
-FROM microsoft/dotnet:2.0-sdk AS build-env
-WORKDIR /app
-
-COPY *.csproj ./
-RUN dotnet restore
-
-COPY . ./
-RUN dotnet publish -c Debug -o out
-
-FROM microsoft/dotnet:2.0-runtime-stretch
-WORKDIR /app
-COPY --from=build-env /app/out ./
-
-RUN apt-get update
-RUN apt-get install -y unzip procps
-RUN curl -sSL https://aka.ms/getvsdbgsh | bash /dev/stdin -v latest -l ~/vsdbg
-
-ENTRYPOINT ["dotnet", "filtermodule.dll"]

iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7

@@ -1,5 +0,0 @@
-FROM microsoft/dotnet:2.0.0-runtime-stretch-arm32v7
-ARG EXE_DIR=.
-WORKDIR /app
-COPY $EXE_DIR/ ./
-CMD ["dotnet", "filtermodule.dll"]

iotedgedev/template/modules/filtermodule/Program.cs

@@ -1,128 +0,0 @@
-namespace filtermodule
-{
-    using System;
-    using System.IO;
-    using System.Runtime.InteropServices;
-    using System.Runtime.Loader;
-    using System.Security.Cryptography.X509Certificates;
-    using System.Text;
-    using System.Threading;
-    using System.Threading.Tasks;
-    using Microsoft.Azure.Devices.Client;
-    using Microsoft.Azure.Devices.Client.Transport.Mqtt;
-
-    class Program
-    {
-        static int counter;
-
-        static void Main(string[] args)
-        {
-            // The Edge runtime gives us the connection string we need -- it is injected as an environment variable
-            string connectionString = Environment.GetEnvironmentVariable("EdgeHubConnectionString");
-
-            // Cert verification is not yet fully functional when using Windows OS for the container
-            bool bypassCertVerification = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
-            if (!bypassCertVerification) InstallCert();
-            Init(connectionString, bypassCertVerification).Wait();
-
-            // Wait until the app unloads or is cancelled
-            var cts = new CancellationTokenSource();
-            AssemblyLoadContext.Default.Unloading += (ctx) => cts.Cancel();
-            Console.CancelKeyPress += (sender, cpe) => cts.Cancel();
-            WhenCancelled(cts.Token).Wait();
-        }
-
-        /// <summary>
-        /// Handles cleanup operations when app is cancelled or unloads
-        /// </summary>
-        public static Task WhenCancelled(CancellationToken cancellationToken)
-        {
-            var tcs = new TaskCompletionSource<bool>();
-            cancellationToken.Register(s => ((TaskCompletionSource<bool>)s).SetResult(true), tcs);
-            return tcs.Task;
-        }
-
-        /// <summary>
-        /// Add certificate in local cert store for use by client for secure connection to IoT Edge runtime
-        /// </summary>
-        static void InstallCert()
-        {
-            string certPath = Environment.GetEnvironmentVariable("EdgeModuleCACertificateFile");
-            if (string.IsNullOrWhiteSpace(certPath))
-            {
-                // We cannot proceed further without a proper cert file
-                Console.WriteLine($"Missing path to certificate collection file: {certPath}");
-                throw new InvalidOperationException("Missing path to certificate file.");
-            }
-            else if (!File.Exists(certPath))
-            {
-                // We cannot proceed further without a proper cert file
-                Console.WriteLine($"Missing path to certificate collection file: {certPath}");
-                throw new InvalidOperationException("Missing certificate file.");
-            }
-            X509Store store = new X509Store(StoreName.Root, StoreLocation.CurrentUser);
-            store.Open(OpenFlags.ReadWrite);
-            store.Add(new X509Certificate2(X509Certificate2.CreateFromCertFile(certPath)));
-            Console.WriteLine("Added Cert: " + certPath);
-            store.Close();
-        }
-
-
-        /// <summary>
-        /// Initializes the DeviceClient and sets up the callback to receive
-        /// messages containing temperature information
-        /// </summary>
-        static async Task Init(string connectionString, bool bypassCertVerification = false)
-        {
-            Console.WriteLine("Connection String {0}", connectionString);
-
-            MqttTransportSettings mqttSetting = new MqttTransportSettings(TransportType.Mqtt_Tcp_Only);
-            // During dev you might want to bypass the cert verification. It is highly recommended to verify certs systematically in production
-            if (bypassCertVerification)
-            {
-                mqttSetting.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true;
-            }
-            ITransportSettings[] settings = { mqttSetting };
-
-            // Open a connection to the Edge runtime
-            DeviceClient ioTHubModuleClient = DeviceClient.CreateFromConnectionString(connectionString, settings);
-            await ioTHubModuleClient.OpenAsync();
-            Console.WriteLine("IoT Hub module client initialized.");
-
-            // Register callback to be called when a message is received by the module
-            await ioTHubModuleClient.SetInputMessageHandlerAsync("input1", PipeMessage, ioTHubModuleClient);
-        }
-
-        /// <summary>
-        /// This method is called whenever the module is sent a message from the EdgeHub. 
-        /// It just pipe the messages without any change.
-        /// It prints all the incoming messages.
-        /// </summary>
-        static async Task<MessageResponse> PipeMessage(Message message, object userContext)
-        {
-            int counterValue = Interlocked.Increment(ref counter);
-
-            var deviceClient = userContext as DeviceClient;
-            if (deviceClient == null)
-            {
-                throw new InvalidOperationException("UserContext doesn't contain " + "expected values");
-            }
-
-            byte[] messageBytes = message.GetBytes();
-            string messageString = Encoding.UTF8.GetString(messageBytes);
-            Console.WriteLine($"Received message: {counterValue}, Body: [{messageString}]");
-
-            if (!string.IsNullOrEmpty(messageString))
-            {
-                var pipeMessage = new Message(messageBytes);
-                foreach (var prop in message.Properties)
-                {
-                    pipeMessage.Properties.Add(prop.Key, prop.Value);
-                }
-                await deviceClient.SendEventAsync("output1", pipeMessage);
-                Console.WriteLine("Received message sent");
-            }
-            return MessageResponse.Completed;
-        }
-    }
-}

iotedgedev/template/modules/filtermodule/filtermodule.csproj

@@ -1,22 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.0</TargetFramework>
-  </PropertyGroup>
-  
-  <PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Debug|netcoreapp2.0|AnyCPU'">
-    <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
-    <TreatSpecificWarningsAsErrors />
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Devices.Client" Version="1.6.0-preview-001" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.0.0-preview2-final" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="2.0.0-preview2-final" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.0.0-preview2-final" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.0.0-preview2-final" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.0-preview2-final" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.0-preview2-final" />
-    <PackageReference Include="System.Runtime.Loader" Version="4.3.0" />
-  </ItemGroup>
-</Project>

iotedgedev/template/modules/filtermodule/module.json

@@ -1,17 +0,0 @@
-{
-    "$schema-version": "0.0.1",
-    "description": "",
-    "image": {
-        "repository": "<registry>/<repo-name>",
-        "tag": {
-            "version": "0.0.1",
-            "platforms": {
-                "amd64": "./Dockerfile",
-                "amd64.debug": "./Dockerfile.amd64.debug",
-                "arm32v7": "./Dockerfile.arm32v7",
-                "windows-amd64": "./Dockerfile"
-            }
-        }
-    },
-    "language": "csharp"
-}

iotedgedev/template/runtime.template.json

@@ -1,7 +1,7 @@
 {
   "deployment": {
     "docker": {
-      "edgeRuntimeImage": "microsoft/azureiotedge-agent:${RUNTIME_TAG}", 
+      "edgeRuntimeImage": "mcr.microsoft.com/azureiotedge-agent:${RUNTIME_TAG}",
       "loggingOptions": {
         "log-driver": "json-file", 
         "log-opts": {

iotedgedev/utility.py

@@ -1,17 +1,19 @@
-from base64 import b64encode, b64decode
 import fnmatch
-from hashlib import sha256
-from hmac import HMAC
 import json
 import os
 import subprocess
 import sys
+from base64 import b64decode, b64encode
+from hashlib import sha256
+from hmac import HMAC
 from time import time
+
+from .moduletype import ModuleType
+
 if sys.version_info.major >= 3:
     from urllib.parse import quote, urlencode
 else:
     from urllib import quote, urlencode
-from .moduletype import ModuleType
 
 
 class Utility:
@@ -78,9 +80,13 @@ class Utility:
 
         return "SharedAccessSignature " + urlencode(rawtoken)
 
-    def get_file_contents(self, file):
+    def get_file_contents(self, file, expand_env=False):
         with open(file, "r") as file:
-            return file.read()
+            content = file.read()
+            if expand_env:
+                return os.path.expandvars(content)
+            else:
+                return content
 
     def decode(self, val):
         return val.decode("utf-8").strip()
@@ -90,9 +96,12 @@ class Utility:
 
         return [os.path.join(os.getcwd(), f) for f in os.listdir(os.getcwd()) if f.endswith("template.json")]
 
-    def get_active_modules(self):
+    def get_bypass_modules(self):
         return [module.strip()
-                for module in self.envvars.ACTIVE_MODULES.split(",") if module]
+                for module in self.envvars.BYPASS_MODULES.split(",") if module]
+
+    def get_active_docker_platform(self):
+        return [platform.strip() for platform in self.envvars.ACTIVE_DOCKER_PLATFORMS.split(",") if platform]
 
     def get_modules_in_config(self, moduleType):
         modules_config = json.load(open(self.envvars.DEPLOYMENT_CONFIG_FILE_PATH))
@@ -112,7 +121,7 @@ class Utility:
             return_modules.update(user_modules)
             return return_modules
 
-    def set_config(self, force=False):
+    def set_config(self, force=False, replacements=None):
 
         if not self.config_set or force:
             self.output.header("PROCESSING CONFIG FILES")
@@ -137,12 +146,34 @@ class Utility:
                 self.output.info("Expanding '{0}' to '{1}'".format(
                     os.path.basename(config_file), build_config_file))
 
-                config_file_expanded = os.path.expandvars(
-                    self.get_file_contents(config_file))
-
-                with open(build_config_file, "w") as config_file_build:
-                    config_file_build.write(config_file_expanded)
+                self.copy_template(config_file, build_config_file, replacements, True)
 
             self.output.line()
 
         self.config_set = True
+
+    def copy_template(self, src, dest=None, replacements=None, expand_env=True):
+        """Read file at src, replace the keys in replacements with their values, optionally expand environment variables, and save to dest"""
+        if dest is None:
+            dest = src
+
+        content = self.get_file_contents(src)
+
+        if replacements:
+            for key, value in replacements.items():
+                content = content.replace(key, value)
+
+        if expand_env:
+            content = os.path.expandvars(content)
+
+        with open(dest, "w") as dest_file:
+            dest_file.write(content)
+
+    def nested_set(self, dic, keys, value):
+        current = dic
+        for key in keys[:-1]:
+            if key not in current:
+                current[key] = {}
+            current = current.get(key)
+
+        current[keys[-1]] = value

requirements.txt

@@ -1,8 +1,7 @@
-docker==3.0.0
+docker>=3.4
 python-dotenv
 requests
 fstrings
-azure-iot-edge-runtime-ctl==1.0.0rc22
 azure-cli-iot
 azure-cli-profile
 azure-cli-extension

setup.cfg

@@ -19,3 +19,5 @@ exclude = docs
 
 [aliases]
 
+[pytest]
+norecursedirs=tests/utility

setup.py

@@ -31,11 +31,10 @@ with open('HISTORY.rst') as history_file:
 
 requirements = [
     'Click>=6.0',
-    'docker==3.0.0',
+    'docker>=3.4',
     'python-dotenv',
     'requests',
     'fstrings',
-    'azure-iot-edge-runtime-ctl==1.0.0rc22',
     'azure-cli-iot',
     'azure-cli-profile',
     'azure-cli-extension',

tests/assets/deployment.template_1.json

@@ -0,0 +1,79 @@
+{
+  "moduleContent": {
+    "$edgeAgent": {
+      "properties.desired": {
+        "schemaVersion": "1.0",
+        "runtime": {
+          "type": "docker",
+          "settings": {
+            "minDockerVersion": "v1.25",
+            "loggingOptions": ""
+          }
+        },
+        "systemModules": {
+          "edgeAgent": {
+            "type": "docker",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-agent:1.0",
+              "createOptions": ""
+            }
+          },
+          "edgeHub": {
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-hub:1.0",
+              "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
+            }
+          }
+        },
+        "modules": {
+          "temp-sensor-module": {
+            "version": "1.0",
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0",
+              "createOptions": ""
+            }
+          },
+          "csharpmodule": {
+            "version": "1.0",
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "${MODULES.csharpmodule.amd64}",
+              "createOptions": ""
+            }
+          },
+          "csharpfunction": {
+            "version": "1.0",
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "${MODULES.csharpfunction.amd64.debug}",
+              "createOptions": ""
+            }
+          }
+        }
+      }
+    },
+    "$edgeHub": {
+      "properties.desired": {
+        "schemaVersion": "1.0",
+        "routes": {
+          "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")",
+          "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream",
+          "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream"
+        },
+        "storeAndForwardConfiguration": {
+          "timeToLiveSecs": 7200
+        }
+      }
+    }
+  }
+}

tests/assets/deployment.template_2.json

@@ -0,0 +1,79 @@
+{
+  "moduleContent": {
+    "$edgeAgent": {
+      "properties.desired": {
+        "schemaVersion": "1.0",
+        "runtime": {
+          "type": "docker",
+          "settings": {
+            "minDockerVersion": "v1.25",
+            "loggingOptions": ""
+          }
+        },
+        "systemModules": {
+          "edgeAgent": {
+            "type": "docker",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-agent:1.0",
+              "createOptions": ""
+            }
+          },
+          "edgeHub": {
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-hub:1.0",
+              "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
+            }
+          }
+        },
+        "modules": {
+          "temp-sensor-module": {
+            "version": "1.0",
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0",
+              "createOptions": ""
+            }
+          },
+          "csharpmodule": {
+            "version": "1.0",
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "localhost:5000/csharpmodule:0.0.1-amd64",
+              "createOptions": ""
+            }
+          },
+          "csharpfunction": {
+            "version": "1.0",
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "localhost:5000/csharpfunction:0.0.1-amd64.debug",
+              "createOptions": ""
+            }
+          }
+        }
+      }
+    },
+    "$edgeHub": {
+      "properties.desired": {
+        "schemaVersion": "1.0",
+        "routes": {
+          "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")",
+          "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream",
+          "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream"
+        },
+        "storeAndForwardConfiguration": {
+          "timeToLiveSecs": 7200
+        }
+      }
+    }
+  }
+}

tests/assets/deployment.template_3.json

@@ -0,0 +1,90 @@
+{
+    "moduleContent": {
+        "$edgeAgent": {
+            "properties.desired": {
+                "schemaVersion": "1.0",
+                "runtime": {
+                    "type": "docker",
+                    "settings": {
+                        "minDockerVersion": "v1.25",
+                        "loggingOptions": ""
+                    }
+                },
+                "systemModules": {
+                    "edgeAgent": {
+                        "type": "docker",
+                        "settings": {
+                            "image": "mcr.microsoft.com/azureiotedge-agent:1.0",
+                            "createOptions": ""
+                        }
+                    },
+                    "edgeHub": {
+                        "type": "docker",
+                        "status": "running",
+                        "restartPolicy": "always",
+                        "settings": {
+                            "image": "mcr.microsoft.com/azureiotedge-hub:1.0",
+                            "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
+                        }
+                    }
+                },
+                "modules": {
+                    "temp-sensor-module": {
+                        "version": "1.0",
+                        "type": "docker",
+                        "status": "running",
+                        "restartPolicy": "always",
+                        "settings": {
+                            "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0",
+                            "createOptions": ""
+                        }
+                    },
+                    "csharpmodule": {
+                        "version": "1.0",
+                        "type": "docker",
+                        "status": "running",
+                        "restartPolicy": "always",
+                        "settings": {
+                            "image": "${MODULES.csharpmodule.amd64}",
+                            "createOptions": ""
+                        }
+                    },
+                    "csharpfunction": {
+                        "version": "1.0",
+                        "type": "docker",
+                        "status": "running",
+                        "restartPolicy": "always",
+                        "settings": {
+                            "image": "${MODULES.csharpfunction.amd64.debug}",
+                            "createOptions": ""
+                        }
+                    },
+                    "csharpmodule2": {
+                        "version": "1.0",
+                        "type": "docker",
+                        "status": "running",
+                        "restartPolicy": "always",
+                        "settings": {
+                            "image": "${MODULES.csharpmodule2.amd64}",
+                            "createOptions": ""
+                        }
+                    }
+                }
+            }
+        },
+        "$edgeHub": {
+            "properties.desired": {
+                "schemaVersion": "1.0",
+                "routes": {
+                    "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")",
+                    "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream",
+                    "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream",
+                    "csharpmodule2ToIoTHub": "FROM /messages/modules/csharpmodule2/outputs/* INTO $upstream"
+                },
+                "storeAndForwardConfiguration": {
+                    "timeToLiveSecs": 7200
+                }
+            }
+        }
+    }
+}

tests/conftest.py

@@ -0,0 +1,4 @@
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "utility"))

tests/node_solution/.env.tmp

@@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="."
 RUNTIME_HOST_NAME="."
     # "." - Auto detect
 
-RUNTIME_TAG="1.0-preview"
+RUNTIME_TAG="1.0"
 
 RUNTIME_VERBOSITY="INFO"
     # "DEBUG", "INFO", "ERROR", "WARNING"

tests/test_azurecli.py

@@ -1,8 +1,10 @@
 import pytest
+
 from iotedgedev.azurecli import get_query_argument_for_id_and_name
 
 pytestmark = pytest.mark.unit
 
+
 def get_terms(query):
     # These tests are all asserting that the query contains two terms enclosed in
     # [?], separated by ||
@@ -10,6 +12,7 @@ def get_terms(query):
     # contract is violated, but we'd want them to fail in that case.
     return query[2:len(query)-1].split(" || ")
 
+
 def test_lowercase_token_should_be_lowercase_for_name_and_id():
     token = "abc123"
     query = get_query_argument_for_id_and_name(token)
@@ -19,6 +22,7 @@ def test_lowercase_token_should_be_lowercase_for_name_and_id():
     assert "starts_with(@.id,'abc123')" in terms
     assert "contains(@.name,'abc123')" in terms
 
+
 def test_mixedcase_token_should_be_lowercase_for_id_but_unmodified_for_name():
     token = "AbC123"
     query = get_query_argument_for_id_and_name(token)

tests/test_connectionstring.py

@@ -1,7 +1,8 @@
-import os
 import pytest
-from dotenv import load_dotenv
-from iotedgedev.connectionstring import ConnectionString, IoTHubConnectionString, DeviceConnectionString
+
+from iotedgedev.connectionstring import (ConnectionString,
+                                         DeviceConnectionString,
+                                         IoTHubConnectionString)
 
 pytestmark = pytest.mark.unit
 
@@ -13,24 +14,29 @@ invalid_connectionstring = "HostName=azure-devices.net;SharedAccessKey=gibberish
 invalid_iothub_connectionstring = "HostName=testhub.azure-devices.net;SharedAccessKey=moregibberish"
 invalid_device_connectionstring = "HostName=testhub.azure-devices.net;DeviceId=;SharedAccessKey=othergibberish"
 
+
 def test_empty_connectionstring():
     connectionstring = ConnectionString(emptystring)
     assert not connectionstring.data
 
+
 def test_empty_iothub_connectionstring():
     connectionstring = IoTHubConnectionString(emptystring)
     assert not connectionstring.data
 
+
 def test_empty_device_connectionstring():
     connectionstring = DeviceConnectionString(emptystring)
     assert not connectionstring.data
 
+
 def test_valid_connectionstring():
     connectionstring = ConnectionString(valid_connectionstring)
     assert connectionstring.HostName == "testhub.azure-devices.net"
     assert connectionstring.HubName == "testhub"
     assert connectionstring.SharedAccessKey == "gibberish"
 
+
 def test_valid_iothub_connectionstring():
     connectionstring = IoTHubConnectionString(valid_iothub_connectionstring)
     assert connectionstring.HostName == "testhub.azure-devices.net"
@@ -38,6 +44,7 @@ def test_valid_iothub_connectionstring():
     assert connectionstring.SharedAccessKeyName == "iothubowner"
     assert connectionstring.SharedAccessKey == "moregibberish"
 
+
 def test_valid_devicehub_connectionstring():
     connectionstring = DeviceConnectionString(valid_device_connectionstring)
     assert connectionstring.HostName == "testhub.azure-devices.net"
@@ -45,14 +52,17 @@ def test_valid_devicehub_connectionstring():
     assert connectionstring.DeviceId == "testdevice"
     assert connectionstring.SharedAccessKey == "othergibberish"
 
+
 def test_invalid_connectionstring():
     connectionstring = ConnectionString(invalid_connectionstring)
     assert connectionstring.HubName != "testhub"
 
+
 def test_invalid_iothub_connectionstring():
     with pytest.raises(KeyError):
         IoTHubConnectionString(invalid_iothub_connectionstring)
 
+
 def test_invalid_devicehub_connectionstring():
     connectionstring = DeviceConnectionString(invalid_device_connectionstring)
     assert connectionstring.HostName == "testhub.azure-devices.net"

tests/test_deploymentmanifest.py

@@ -0,0 +1,50 @@
+import json
+import os
+
+import pytest
+
+from iotedgedev.deploymentmanifest import DeploymentManifest
+from iotedgedev.envvars import EnvVars
+from iotedgedev.output import Output
+from iotedgedev.utility import Utility
+from utility import assert_list_equal
+
+pytestmark = pytest.mark.unit
+
+tests_dir = os.path.join(os.getcwd(), "tests")
+test_assets_dir = os.path.join(tests_dir, "assets")
+test_file_1 = os.path.join(test_assets_dir, "deployment.template_1.json")
+test_file_2 = os.path.join(test_assets_dir, "deployment.template_2.json")
+test_file_3 = os.path.join(test_assets_dir, "deployment.template_3.json")
+
+
+@pytest.fixture
+def deployment_manifest():
+    output = Output()
+    envvars = EnvVars(output)
+    envvars.load()
+    utility = Utility(envvars, output)
+
+    def _deployment_manifest(path):
+        return DeploymentManifest(envvars, output, utility, path, True)
+
+    return _deployment_manifest
+
+
+def test_get_modules_to_process(deployment_manifest):
+    deployment_manifest = deployment_manifest(test_file_1)
+    modules_to_process = deployment_manifest.get_modules_to_process()
+    assert_list_equal(modules_to_process, [("csharpmodule", "amd64"), ("csharpfunction", "amd64.debug")])
+
+
+def test_get_modules_to_process_empty(deployment_manifest):
+    deployment_manifest = deployment_manifest(test_file_2)
+    modules_to_process = deployment_manifest.get_modules_to_process()
+    assert_list_equal(modules_to_process, [])
+
+
+def test_add_module_template(deployment_manifest):
+    deployment_manifest = deployment_manifest(test_file_1)
+    deployment_manifest.add_module_template("csharpmodule2")
+    with open(test_file_3, "r") as expected:
+        assert deployment_manifest.json == json.load(expected)

tests/test_envvars.py

@@ -5,24 +5,28 @@ from iotedgedev.output import Output
 
 pytestmark = pytest.mark.unit
 
+
 def test_valid_get_envvar():
     output = Output()
     envvars = EnvVars(output)
     loglevel = envvars.get_envvar("RUNTIME_LOG_LEVEL")
     assert loglevel == "info" or "debug"
 
+
 def test_invalid_get_envvar():
     output = Output()
     envvars = EnvVars(output)
     testerval = envvars.get_envvar("TESTER")
     assert not testerval
 
+
 def test_valid_load():
     output = Output()
     envvars = EnvVars(output)
     envvars.load()
     assert envvars.RUNTIME_LOG_LEVEL == "info" or "debug"
 
+
 def test_valid_verify_envvar_has_val():
     output = Output()
     envvars = EnvVars(output)
@@ -30,16 +34,19 @@ def test_valid_verify_envvar_has_val():
     result = envvars.verify_envvar_has_val("RUNTIME_LOG_LEVEL", envvars.RUNTIME_LOG_LEVEL)
     assert not result
 
+
 def test_valid_get_envvar_key_if_val():
     output = Output()
     envvars = EnvVars(output)
     assert envvars.get_envvar_key_if_val("RUNTIME_LOG_LEVEL")
 
+
 def test_invalid_get_envvar_key_if_val():
     output = Output()
     envvars = EnvVars(output)
     assert not envvars.get_envvar_key_if_val("TESTER")
 
+
 def test_set_envvar():
     output = Output()
     envvars = EnvVars(output)
@@ -48,4 +55,3 @@ def test_set_envvar():
     setlevel = envvars.get_envvar("RUNTIME_LOG_LEVEL")
     assert setlevel == "debug"
     envvars.set_envvar("RUNTIME_LOG_LEVEL", loglevel)
-    

tests/test_iotedgedev.py

@@ -1,9 +1,13 @@
 import json
 import os
 import shutil
-import pytest
 
+import pytest
 from click.testing import CliRunner
+from dotenv import load_dotenv
+
+from iotedgedev.connectionstring import (DeviceConnectionString,
+                                         IoTHubConnectionString)
 
 pytestmark = pytest.mark.e2e
 
@@ -106,7 +110,7 @@ def test_module_add():
     runner = CliRunner()
 
     add_module_and_verify(cli.main, runner, "csharp")
-    # add_module_and_verify(cli.main, runner, "nodejs")
+    add_module_and_verify(cli.main, runner, "nodejs")
     add_module_and_verify(cli.main, runner, "python")
     add_module_and_verify(cli.main, runner, "csharpfunction")
 
@@ -161,49 +165,49 @@ def test_deploy_modules(request):
     assert 'DEPLOYMENT COMPLETE' in result.output
 
 
-@pytest.fixture
-def test_start_runtime(request):
+# @pytest.fixture
+# def test_start_runtime(request):
 
-    os.chdir(test_solution_dir)
+#     os.chdir(test_solution_dir)
 
-    cli = __import__("iotedgedev.cli", fromlist=['main'])
-    runner = CliRunner()
-    result = runner.invoke(cli.main, ['start'])
-    print(result.output)
+#     cli = __import__("iotedgedev.cli", fromlist=['main'])
+#     runner = CliRunner()
+#     result = runner.invoke(cli.main, ['start'])
+#     print(result.output)
 
-    assert 'Runtime started' in result.output
+#     assert 'Runtime started' in result.output
 
 
-@pytest.fixture
-def test_monitor(request, capfd):
+# @pytest.fixture
+# def test_monitor(request, capfd):
 
-    os.chdir(test_solution_dir)
+#     os.chdir(test_solution_dir)
 
-    cli = __import__("iotedgedev.cli", fromlist=['main'])
-    runner = CliRunner()
-    result = runner.invoke(cli.main, ['monitor', '--timeout', '60000'])
-    out, err = capfd.readouterr()
-    print(out)
-    print(err)
-    print(result.output)
+#     cli = __import__("iotedgedev.cli", fromlist=['main'])
+#     runner = CliRunner()
+#     result = runner.invoke(cli.main, ['monitor', '--timeout', '60000'])
+#     out, err = capfd.readouterr()
+#     print(out)
+#     print(err)
+#     print(result.output)
 
-    assert 'timeCreated' in out
+#     assert 'timeCreated' in out
 
 
-@pytest.fixture
-def test_stop(request):
+# @pytest.fixture
+# def test_stop(request):
 
-    os.chdir(test_solution_dir)
+#     os.chdir(test_solution_dir)
 
-    cli = __import__("iotedgedev.cli", fromlist=['main'])
-    runner = CliRunner()
-    result = runner.invoke(cli.main, ['stop'])
-    print(result.output)
+#     cli = __import__("iotedgedev.cli", fromlist=['main'])
+#     runner = CliRunner()
+#     result = runner.invoke(cli.main, ['stop'])
+#     print(result.output)
 
-    assert 'Runtime stopped' in result.output
+#     assert 'Runtime stopped' in result.output
 
 
-def test_e2e(test_push_modules, test_deploy_modules, test_start_runtime, test_monitor, test_stop):
+def test_e2e(test_push_modules, test_deploy_modules):
     print('Testing E2E')
 
 
@@ -219,9 +223,10 @@ def setup_node_solution(request):
     return
 
 
-def test_node(setup_node_solution, test_push_modules, test_deploy_modules, test_start_runtime, test_monitor, test_stop):
+def test_node(setup_node_solution, test_push_modules, test_deploy_modules):
     print('Testing Node Solution')
 
+
 def test_valid_env_iothub_connectionstring():
     load_dotenv(".env")
     env_iothub_connectionstring = os.getenv("IOTHUB_CONNECTION_STRING")
@@ -231,6 +236,7 @@ def test_valid_env_iothub_connectionstring():
     assert connectionstring.SharedAccessKey
     assert connectionstring.SharedAccessKeyName
 
+
 def test_valid_env_device_connectionstring():
     load_dotenv(".env")
     env_device_connectionstring = os.getenv("DEVICE_CONNECTION_STRING")
@@ -240,6 +246,7 @@ def test_valid_env_device_connectionstring():
     assert connectionstring.SharedAccessKey
     assert connectionstring.DeviceId
 
+
 '''
 def test_load_no_dotenv():
 

tests/test_utility.py

@@ -0,0 +1,44 @@
+import os
+
+import pytest
+
+from iotedgedev.envvars import EnvVars
+from iotedgedev.output import Output
+from iotedgedev.utility import Utility
+from utility import assert_json_file_equal
+
+pytestmark = pytest.mark.unit
+
+tests_dir = os.path.join(os.getcwd(), "tests")
+test_assets_dir = os.path.join(tests_dir, "assets")
+test_file_1 = os.path.join(test_assets_dir, "deployment.template_1.json")
+test_file_2 = os.path.join(test_assets_dir, "deployment.template_2.json")
+
+
+@pytest.fixture
+def utility():
+    output = Output()
+    envvars = EnvVars(output)
+    envvars.load()
+    return Utility(envvars, output)
+
+
+def test_copy_template(utility, tmpdir):
+    replacements = {
+        "${MODULES.csharpmodule.amd64}": "localhost:5000/csharpmodule:0.0.1-amd64",
+        "${MODULES.csharpfunction.amd64.debug}": "localhost:5000/csharpfunction:0.0.1-amd64.debug"
+    }
+    dest = tmpdir.join("deployment_template_1.dest.json").strpath
+    utility.copy_template(test_file_1, dest, replacements=replacements, expand_env=False)
+    assert_json_file_equal(test_file_2, dest)
+
+
+def test_copy_template_expand_env(utility, tmpdir):
+    replacements = {
+        "${MODULES.csharpmodule.amd64}": "${CONTAINER_REGISTRY_SERVER}/csharpmodule:0.0.1-amd64",
+        "${MODULES.csharpfunction.amd64.debug}": "${CONTAINER_REGISTRY_SERVER}/csharpfunction:0.0.1-amd64.debug"
+    }
+    os.environ["CONTAINER_REGISTRY_SERVER"] = "localhost:5000"
+    dest = tmpdir.join("deployment_template_2.dest.json").strpath
+    utility.copy_template(test_file_1, dest, replacements=replacements, expand_env=True)
+    assert_json_file_equal(test_file_2, dest)

tests/utility/utility.py

@@ -0,0 +1,11 @@
+import json
+
+
+def assert_list_equal(list1, list2):
+    return len(list1) == len(list2) and sorted(list1) == sorted(list2)
+
+
+def assert_json_file_equal(file1, file2):
+    with open(file1, "r") as f1:
+        with open(file2, "r") as f2:
+            assert json.load(f1) == json.load(f2)