Adjust resource processor to correctly process a new resource message (#201)

Adjust resource processor to correctly process a new resource message and updates CNAB container definition

.flake8

@@ -1,2 +1,2 @@
 [flake8]
-ignore = E501
+ignore = E501, W503

.github/linters/.flake8

@@ -1,2 +1,2 @@
 [flake8]
-ignore = E501
+ignore = E501,W503

CNAB_container/Dockerfile

@@ -1,15 +1,16 @@
-FROM debian:latest
-
-RUN apt-get update -y && \
-    apt-get upgrade -y && \
-    apt-get install -y curl jq
+FROM mcr.microsoft.com/azure-cli
 
 ARG USER=porter
-RUN useradd -ms /bin/bash $USER
+RUN apk add --update docker openrc
+RUN rc-update add docker boot
+RUN adduser -S $USER -s /bin/bash
 USER $USER
 
+COPY azure.json /home/$USER/
+
 # Install Porter & Azure CNAB driver.
 ENV ACC_CLOUD='true'
 RUN curl https://raw.githubusercontent.com/deislabs/cnab-azure-driver/main/install-in-azure-cloudshell.sh | /bin/bash
+RUN source ${HOME}/.bashrc
 ENV ACC_CLOUD=
 ENV PATH="/home/$USER/.porter:/home/$USER/.cnab-azure-driver:$PATH"

CNAB_container/azure.json

@@ -0,0 +1,32 @@
+{
+    "schemaVersion": "1.0.0-DRAFT+b6c701f",
+    "name": "azure",
+    "created": "2021-06-03T11:31:05.7314113Z",
+    "modified": "2021-06-03T11:31:05.7314113Z",
+    "credentials": [
+      {
+        "name": "azure_client_id",
+        "source": {
+          "env": "ARM_CLIENT_ID"
+        }
+      },
+      {
+        "name": "azure_client_secret",
+        "source": {
+          "env": "ARM_CLIENT_SECRET"
+        }
+      },
+      {
+        "name": "azure_subscription_id",
+        "source": {
+          "env": "ARM_SUBSCRIPTION_ID"
+        }
+      },
+      {
+        "name": "azure_tenant_id",
+        "source": {
+          "env": "ARM_TENANT_ID"
+        }
+      }
+    ]
+  }

processor_function/shared/cnab_builder.py

@@ -5,18 +5,15 @@ import logging
 from typing import List
 
 from azure.mgmt.containerinstance import ContainerInstanceManagementClient
-from azure.identity import DefaultAzureCredential
 from azure.mgmt.containerinstance.models import (ContainerGroup,
                                                  Container,
                                                  ContainerGroupRestartPolicy,
-                                                 ContainerGroupNetworkProfile,
                                                  EnvironmentVariable,
                                                  ImageRegistryCredential,
                                                  ContainerGroupIdentity,
                                                  ResourceRequests,
                                                  ResourceRequirements,
                                                  OperatingSystemTypes)
-from azure.mgmt.network import NetworkManagementClient
 
 from shared.azure_identity_credential_adapter import AzureIdentityCredentialAdapter
 
@@ -29,56 +26,65 @@ class CNABBuilder:
         self._container_group_name = ""
         self._location = ""
 
-    def _build_porter_cmd_line(self) -> str:
+    def _build_porter_command(self) -> List[str]:
         porter_parameters = ""
         for key in self._message['parameters']:
             porter_parameters += " --param " + key + "=" + self._message['parameters'][key]
 
-        start_command_line = "/bin/bash -c porter " + self._message['operation'] + " CNAB --tag " + self._message[
-            'bundle-name'] + porter_parameters + " -d azure && porter show CNAB"
-
-        logging.info("Creating a runner with:" + start_command_line)
-        return start_command_line
-
-    def _build_cnab_env_variables(self) -> List[str]:
-        env_variables = []
         for key, value in os.environ.items():
-            if key.startswith("CNAB_AZURE"):
-                env_variables.append(EnvironmentVariable(name=key, value=value))
+            if key.startswith("param_"):
+                porter_parameters += " --param " + key.replace('param_', '') + "=" + value
+
+        installation_id = self._message['parameters']['tre_id'] + "-" + self._message['parameters']['workspace_id']
+        command_line = ["/bin/bash", "-c", "az login --identity "
+                        + "&& TOKEN=$(az acr login --name msfttreacr --expose-token --output tsv --query accessToken) "
+                        + "&& docker login msfttreacr.azurecr.io --username 00000000-0000-0000-0000-000000000000 --password $TOKEN "
+                        + "&& porter "
+                        + self._message['action'] + " "
+                        + installation_id
+                        + " --reference "
+                        + os.environ["REGISTRY_SERVER"]
+                        + os.environ["WORKSPACES_PATH"]
+                        + self._message['name'] + ":"
+                        + "v" + self._message['version'] + " "
+                        + porter_parameters
+                        + " --cred ./home/porter/azure.json"
+                        + " --driver azure && porter show " + installation_id]
+
+        return command_line
+
+    @staticmethod
+    def _get_environment_variable(key, value):
+        if key.startswith("CNAB_AZURE"):
+            return EnvironmentVariable(name=key, value=value)
+        if key.startswith("SEC_"):
+            return EnvironmentVariable(name=key.replace("SEC_", ""), secure_value=value)
+
+    def _build_cnab_env_variables(self) -> List[EnvironmentVariable]:
+        env_variables = [self._get_environment_variable(key, value) for key, value in os.environ.items()
+                         if key.startswith("CNAB_AZURE") or key.startswith("SEC_")]
 
         env_variables.append(EnvironmentVariable(name="CNAB_AZURE_RESOURCE_GROUP", value=self._resource_group_name))
         env_variables.append(EnvironmentVariable(name="CNAB_AZURE_LOCATION", value=self._location))
 
         return env_variables
 
-    def _get_network_profile(self) -> ContainerGroupNetworkProfile:
-        default_credential = DefaultAzureCredential()
-
-        network_client = NetworkManagementClient(default_credential, self._subscription_id)
-
-        net_results = network_client.network_profiles.list(resource_group_name=self._resource_group_name)
-
-        if net_results:
-            net_result = net_results.next()
-        else:
-            raise Exception("No network profile")
-
-        network_profile = ContainerGroupNetworkProfile(id=net_result.id)
-        return network_profile
-
     def _setup_aci_deployment(self) -> ContainerGroup:
+        """
+        Prepares a Container Group and a Container for deployment to ACI
+        :return: The prepared container group
+        """
 
-        self._location = self._message['parameters']['location']
+        self._location = self._message['parameters']['azure_location']
         self._container_group_name = "aci-cnab-" + str(uuid.uuid4())
-        container_image_name = self._message['CNAB-image']
+        container_image_name = os.environ['CNAB_IMAGE']
 
         image_registry_credentials = [ImageRegistryCredential(server=os.environ["REGISTRY_SERVER"],
-                                                              username=os.environ["REGISTRY_USER_NAME"],
-                                                              password=os.environ["REGISTRY_USER_PASSWORD"])]
+                                                              username=os.environ["SEC_CNAB_AZURE_REGISTRY_USERNAME"],
+                                                              password=os.environ["SEC_CNAB_AZURE_REGISTRY_PASSWORD"])]
 
         managed_identity = ContainerGroupIdentity(type='UserAssigned',
-                                                  user_assigned_identities={
-                                                      os.environ["CNAB_AZURE_USER_MSI_RESOURCE_ID"]: {}})
+                                                  user_assigned_identities={os.environ["CNAB_AZURE_USER_MSI_RESOURCE_ID"]: {}})
 
         container_resource_requests = ResourceRequests(memory_in_gb=1, cpu=1.0)
         container_resource_requirements = ResourceRequirements(requests=container_resource_requests)
@@ -86,24 +92,29 @@ class CNABBuilder:
         container = Container(name=self._container_group_name,
                               image=container_image_name,
                               resources=container_resource_requirements,
-                              command=self._build_porter_cmd_line().split(),
+                              command=self._build_porter_command(),
                               environment_variables=self._build_cnab_env_variables())
 
         group = ContainerGroup(location=self._location,
                                containers=[container],
                                image_registry_credentials=image_registry_credentials,
                                os_type=OperatingSystemTypes.linux,
-                               network_profile=self._get_network_profile(),
                                restart_policy=ContainerGroupRestartPolicy.never,
                                identity=managed_identity)
+
         return group
 
     def deploy_aci(self):
+        """
+        Deploys a CNAB container into ACI with parameters to run porter
+        """
+
         group = self._setup_aci_deployment()
 
         credential = AzureIdentityCredentialAdapter()
         aci_client = ContainerInstanceManagementClient(credential, self._subscription_id)
-        result = aci_client.container_groups.create_or_update(self._resource_group_name, self._container_group_name, group)
+        result = aci_client.container_groups.create_or_update(self._resource_group_name, self._container_group_name,
+                                                              group)
 
         while result.done() is False:
             logging.info('-- Deploying -- ' + self._container_group_name + " to " + self._resource_group_name)

processor_function/test_processor_function/test_cnab_builder.py

@@ -0,0 +1,24 @@
+import os
+from mock import patch
+
+from azure.mgmt.containerinstance.models import EnvironmentVariable
+
+from shared.cnab_builder import CNABBuilder
+
+
+@patch.dict(os.environ, {"CNAB_AZURE_MSI_TYPE": "msi-type",
+                         "CNAB_AZURE_SUBSCRIPTION_ID": "sub-id",
+                         "SEC_ARM_CLIENT_ID": "some-key",
+                         "CNAB_IMAGE": "some value",
+                         "SOMETHING_ELSE": "whatever",
+                         "RESOURCE_GROUP_NAME": "rg-name"})
+def test_build_cnab_env_variables_builds_the_correct_variables():
+    builder = CNABBuilder({})
+
+    actual_env_variables = builder._build_cnab_env_variables()
+    assert len(actual_env_variables) == 5
+    assert EnvironmentVariable(name="CNAB_AZURE_MSI_TYPE", value="msi-type") in actual_env_variables
+    assert EnvironmentVariable(name="CNAB_AZURE_SUBSCRIPTION_ID", value="sub-id") in actual_env_variables
+    assert EnvironmentVariable(name="ARM_CLIENT_ID", secure_value="some-key") in actual_env_variables
+    assert EnvironmentVariable(name="CNAB_AZURE_RESOURCE_GROUP", value="rg-name") in actual_env_variables
+    assert EnvironmentVariable(name="CNAB_AZURE_LOCATION", value="") in actual_env_variables

requirements.txt

@@ -20,6 +20,9 @@ mock==4.0.3
 
 # Requirements for processor function
 azure-functions
+azure-mgmt-containerinstance
+azure-mgmt-network
+msrest
 
 # Requirements for service bus send sample
 azure-functions

samples/servicebus_send_sample/createWorkspace.json

@@ -7,7 +7,7 @@
         "azure_location": "westeurope",
         "tre_id":"9075",
         "core_name": "msfttre-dev-9075",
-        "workspace_id":"2",
+        "workspace_id":"0004",
         "address_space":"10.2.1.0/24"
       }
   }

samples/servicebus_send_sample/send_message_to_servicebus.py

@@ -1,7 +1,9 @@
+import os
+
 from azure.servicebus import ServiceBusClient, ServiceBusMessage
 
 
-CONNECTION_STR = ""
+CONNECTION_STR = os.environ['SERVICEBUSCONNECTION']
 QUEUE_NAME = "workspacequeue"
 
 

scripts/bootstrap_db_for_local_tests.py

@@ -1,6 +1,6 @@
 """
 This script adds resources to your cosmos database for local, manual, testing purposes
-NOTE: This script is not required for the tests to run, but assists in creating resources to test the API manually
+NOTE: This script is not required for the test_processor_function to run, but assists in creating resources to test the API manually
 """
 import uuid
 

templates/core/terraform/processor_function/function.tf

@@ -25,8 +25,8 @@ resource "azurerm_function_app" "procesorfunction" {
     CNAB_AZURE_VERBOSE                    = "true"
     CNAB_AZURE_PROPAGATE_CREDENTIALS      = "true"
     CNAB_AZURE_MSI_TYPE                   = "user"
-    REGISTRY_USER_NAME                    = var.docker_registry_username
-    REGISTRY_USER_PASSWORD                = var.docker_registry_password
+    CNAB_AZURE_REGISTRY_USERNAME          = var.docker_registry_username
+    CNAB_AZURE_REGISTRY_PASSWORD          = var.docker_registry_password
     REGISTRY_SERVER                       = var.docker_registry_server
     servicebusconnection                  = var.servicebus_connection_string
     queueName                             = var.workspacequeue

workspaces/vanilla/.env.sample

@@ -5,7 +5,7 @@ ARM_CLIENT_SECRET=_CHANGEME_
 tre_id="mytre-dev-3142"
 workspace_id="0a9e"
 address_space="10.2.6.0/24"
-location=westeurope
+azure_location=westeurope
 TF_VAR_tfstate_resource_group_name=_CHANGEME_
 TF_VAR_tfstate_storage_account_name=_CHANGEME_
 TF_VAR_tfstate_container_name=_CHANGEME_

workspaces/vanilla/parameters.json

@@ -11,9 +11,9 @@
       }
     },
     {
-      "name": "location",
+      "name": "azure_location",
       "source": {
-        "env": "LOCATION"
+        "env": "AZURE_LOCATION"
       }
     },
     {

workspaces/vanilla/porter.yaml

@@ -22,7 +22,7 @@ parameters:
     description: "Unique 4-character long, alphanumeric workspace ID"
     minLength: 4
     maxLength: 4
-  - name: location
+  - name: azure_location
     type: string
     description: "Azure location (region) to deploy to"
   - name: address_space
@@ -52,7 +52,7 @@ install:
       vars:
         tre_id: "{{ bundle.parameters.tre_id }}"
         workspace_id: "{{ bundle.parameters.workspace_id }}"
-        location: "{{ bundle.parameters.location }}"
+        location: "{{ bundle.parameters.azure_location }}"
         address_space: "{{ bundle.parameters.address_space }}"
       backendConfig:
         resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}"
@@ -74,7 +74,7 @@ uninstall:
       vars:
         tre_id: "{{ bundle.parameters.tre_id }}"
         workspace_id: "{{ bundle.parameters.workspace_id }}"
-        location: "{{ bundle.parameters.location }}"
+        location: "{{ bundle.parameters.azure_location }}"
         address_space: "{{ bundle.parameters.address_space }}"
       backendConfig:
         resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}"