Add "mesh deployment create" command (#146)

2018-11-09 05:36:09 +05:30 · 2018-11-09 05:36:09 +05:30 · 48868217b2
--- a/.gitignore
+++ b/.gitignore
@ -108,5 +108,8 @@ ENV/
 # OS Files
 .DS_Store

-# Ignore recordings to keep tests local
-recordings
+# Ignore "recordings" folder, which is generated by tests using the vcr module
+recordings
+
+# Ignore "meshDeploy" folder as this is used to store the generated JSON files for resource deployment
+meshDeploy
--- a/11
+++ b/11
@ -50,3 +50,14 @@ src/sfctl/helps/cluster_upgrade.py @jkochhar

 # Version
 src/sfctl/custom_version.py @jeffj6123
+
+# Mesh Deployment
+src/sfctl/custom_deployment.py @rapatchi @ashank
+src/sfctl/helps/deployment.py @rapatchi @ashank
+src/sfctl/tests/mesh_test.py @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_app.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_gateway.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_network.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_secret.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_secret_value.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_volume.yaml @rapatchi @ashank
--- a/2
+++ b/2
@ -460,7 +460,7 @@ max-bool-expr=5
 max-branches=12

 # Maximum number of locals for function / method body
-max-locals=15
+max-locals=20

 # Maximum number of parents for a class (see R0901).
 max-parents=7
--- a/src/README.rst
+++ b/src/README.rst
@ -25,6 +25,7 @@ Unreleased
 - Add Mesh network, gateway, code package, secret, and secretvalue commands (#141)
 - Allow any Python 3.7.x versions rather than only 3.7.0 (#142)
 - Fix missing option of "Error" health state in health reporting (#151)
+- Add the command "sfctl mesh deployment create", which takes resource description yaml files as input and deploys the corresponding mesh resources (#146)

 6.0.1
 -----
--- a/src/setup.py
+++ b/src/setup.py
@ -51,7 +51,8 @@ setup(
        'azure-servicefabric==6.3.0.0',
        'jsonpickle',
        'adal',
-        'future'
+        'future',
+        'sfmergeutility'
    ],
    extras_require={
        'test': [
--- a/src/sfctl/commands.py
+++ b/src/sfctl/commands.py
@ -30,6 +30,8 @@ import sfctl.helps.app_type  # pylint: disable=unused-import
 import sfctl.helps.chaos  # pylint: disable=unused-import
 import sfctl.helps.infrastructure  # pylint: disable=unused-import
 import sfctl.helps.secretvalue  # pylint: disable=unused-import
+import sfctl.helps.deployment # pylint: disable=unused-import
+
 EXCLUDED_PARAMS = ['self', 'raw', 'custom_headers', 'operation_config',
                   'content_version', 'kwargs', 'client']

@ -400,6 +402,11 @@ class SFCommandLoader(CLICommandsLoader):
                          client_factory=mesh_secret_value_create) as group:
            group.command('show', 'get_secret_value')

+        client_func_path_mesh = 'sfctl.custom_deployment#{}'
+        with CommandGroup(self, 'mesh deployment', client_func_path_mesh,
+                          client_factory=client_create) as group:
+            group.command('create', 'mesh_deploy')
+
        return OrderedDict(self.command_table)

    def load_arguments(self, command):
--- a/src/sfctl/custom_deployment.py
+++ b/src/sfctl/custom_deployment.py
@ -0,0 +1,77 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+"""Custom commands for managing Service Fabric Mesh resources"""
+
+from __future__ import print_function
+
+import os
+import shutil
+from knack.util import CLIError
+from sfmergeutility import SFMergeUtility
+from sfmergeutility.utility import ResourceType, get_resource_name, get_resource_type, list_files_in_directory, load_json # pylint: disable=line-too-long
+
+def deploy_resource(client, resource):
+    """ Deploys the specified resource to the connected cluster
+    :param client: (class) Auto generated client from swagger specification
+    :param resource: (str) Relative/absolute path of the resource file
+    """
+    resource_type = get_resource_type(resource)
+    resource_name = get_resource_name(resource)
+
+    print('Creating resource: ', resource_name, 'of type: ', resource_type.name)
+
+    if resource_type == ResourceType.application:
+        application_description = load_json(resource)
+        client.mesh_application.create_or_update(resource_name, application_description.get('description')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.volume:
+        volume_description = load_json(resource)
+        client.mesh_volume.create_or_update(resource_name, volume_description.get('description')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.network:
+        network_description = load_json(resource)
+        client.mesh_network.create_or_update(resource_name, network_description.get('description').get('name'), network_description.get('description').get('properties')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.secret:
+        secret_description = load_json(resource)
+        client.mesh_secret.create_or_update(resource_name, secret_description.get('description').get('properties'), secret_description.get('description').get('name')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.secretValue:
+        secret_value_description = load_json(resource)
+        fully_qualified_resource_name = secret_value_description.get('fullyQualifiedResourceName').split('/') # pylint: disable=line-too-long
+        secret_value_resource_name = fully_qualified_resource_name[1]
+        client.mesh_secret_value.add_value(resource_name, secret_value_resource_name, secret_value_description.get('description').get('name'), secret_value_description.get('description').get('properties').get('value')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.gateway:
+        gateway_description = load_json(resource)
+        client.mesh_gateway.create_or_update(resource_name, gateway_description.get('description')) # pylint: disable=line-too-long
+
+def mesh_deploy(client, input_yaml_file_paths, parameters=None):
+    """ This function
+        1. Uses sfmergeutility to merge, convert, and
+        order the resources
+        2. Deploys the resources in the order suggested by the utility
+    :param client: (class) Auto generated client from swagger specification
+    :param input_yaml_file_paths: (str) Relative/absolute directory path or comma seperated relative/absolute file paths of the yaml resource files  # pylint: disable=line-too-long
+    """
+    file_path_list = []
+
+    if os.path.isdir(input_yaml_file_paths):
+        if not os.path.exists(input_yaml_file_paths):
+            raise CLIError('The specified directory "%s" does not exist or you do not have access to it' %(input_yaml_file_paths)) # pylint: disable=line-too-long
+        file_path_list = list_files_in_directory(input_yaml_file_paths, ".yaml")
+
+    else:
+        file_path_list = input_yaml_file_paths.split(',')
+        for file_path in file_path_list:
+            if not os.path.exists(file_path):
+                raise CLIError('The specified file "%s" does not exist or you do not have access to it' %(file_path)) # pylint: disable=line-too-long
+
+    output_dir = os.path.join(os.getcwd(), "meshDeploy")
+    if os.path.exists(output_dir):
+        shutil.rmtree(output_dir, ignore_errors=True)
+
+    SFMergeUtility.sf_merge_utility(file_path_list, "SF_SBZ_JSON", parameter_file=parameters, output_dir=output_dir, prefix="") # pylint: disable=line-too-long
+    resources = list_files_in_directory(output_dir, ".json")
+    resources.sort()
+    for resource in resources:
+        deploy_resource(client, resource)
--- a/src/sfctl/helps/deployment.py
+++ b/src/sfctl/helps/deployment.py
@ -0,0 +1,26 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+"""Help documentation for managing Service Fabric Mesh Resources."""
+
+from knack.help_files import helps
+
+helps['mesh deployment create'] = """
+    type: command
+    short-summary: Creates a deployment of Service Fabric Mesh Resources
+    parameters:
+        - name: --input-yaml-file-paths
+          type: string
+          short-summary: Comma separated relative/absolute file paths of all the yaml files or relative/absolute path of the directory (recursive) which contain yaml files
+        - name: --parameters
+          type: string
+          short-summary: A relative/absolute to json file which contains the parameters that need to be overridden
+    examples:
+        - name: Consolidates and deploys all the resources to cluster by overriding the parameters mentioned in the param.json file
+          text: sfctl mesh deployment create --input-yaml-file-paths ./app.yaml,./network.yaml --parameters ./param.json
+        - name: Consolidates and deploys all the resources in a directory to cluster by overriding the parameters mentioned in the param.json file
+          text: sfctl mesh deployment create --input-yaml-file-paths ./resources --parameters ./param.json
+"""
--- a/src/sfctl/helps/main.py
+++ b/src/sfctl/helps/main.py
@ -29,7 +29,7 @@ helps['sa-cluster'] = """

 helps['application'] = """
    type: group
-    short-summary: Create, delete, and manage applications and application types.
+    short-summary: Create, delete, and manage applications and application types
 """

 helps['chaos'] = """
@ -104,6 +104,11 @@ helps['mesh app'] = """
    short-summary: Get and delete application resources
 """

+helps['mesh deployment'] = """
+    type: group
+    short-summary: Create Service Fabric Mesh resources
+"""
+
 helps['mesh service'] = """
    type: group
    short-summary: Get service details and list services of an application resource
--- a/src/sfctl/params.py
+++ b/src/sfctl/params.py
@ -209,8 +209,3 @@ def custom_arguments(self, _):  # pylint: disable=too-many-statements
        # expect the parameter command_input in the python method as --command in commandline.
        arg_context.argument('command_input',
                             CLIArgumentType(options_list='--command'))
-
-    with ArgumentsContext(self, 'mesh secretvalue show') as arg_context:
-        arg_context.argument('secret_resource_name')
-        arg_context.argument('secret_value_resource_name')
-        arg_context.argument('show_value')
--- a/src/sfctl/tests/custom_help_text_correctness_test.py
+++ b/src/sfctl/tests/custom_help_text_correctness_test.py
@ -224,7 +224,7 @@ class CustomHelpTextCorrectnessTests(unittest.TestCase):
            print()
            print(line)

-        allowable_lines_not_found = 75
+        allowable_lines_not_found = 78

        print()
        print('The total number of lines compared is ' + str(len(custom_help_lines)))
--- a/src/sfctl/tests/help_text_test.py
+++ b/src/sfctl/tests/help_text_test.py
@ -312,16 +312,21 @@ class HelpTextTests(unittest.TestCase):
            commands=('delete', 'list', 'show'))

        self.validate_output(
-            'sfctl mesh volume',
-            commands=('delete', 'list', 'show'))
+            'sfctl mesh deployment',
+            commands=('create',))

        self.validate_output(
            'sfctl mesh service',
            commands=('list', 'show'))
+
        self.validate_output(
            'sfctl mesh service-replica',
            commands=('list', 'show'))

+        self.validate_output(
+            'sfctl mesh volume',
+            commands=('delete', 'list', 'show'))
+
        self.validate_output(
            'sfctl node',
            commands=('disable', 'enable', 'health', 'info', 'list', 'load', 'remove-state',
--- a/src/sfctl/tests/mesh_test.py
+++ b/src/sfctl/tests/mesh_test.py
@ -0,0 +1,21 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+"""Custom mesh command tests"""
+
+import unittest
+from knack.util import CLIError
+import sfctl.custom_deployment as sf_resource
+
+class MeshTests(unittest.TestCase):
+    """Mesh command tests """
+
+    def test_mesh_deploy_invalid(self):
+        """ Test to check if mesh deployment command fails when invalid path is provided"""
+        with self.assertRaises(CLIError):
+            sf_resource.mesh_deploy(None, "some-dummy-file-path")
+        with self.assertRaises(CLIError):
+            sf_resource.mesh_deploy(None, "some-dummy-file-path,another-dummy-file-path")
--- a/src/sfctl/tests/request_generation_test.py
+++ b/src/sfctl/tests/request_generation_test.py
@ -218,12 +218,13 @@ class ServiceFabricRequestTests(ScenarioTest):
        # Call test
        self.paths_generation_helper()

-    def paths_generation_helper(self):  # pylint: disable=too-many-statements
+    def paths_generation_helper(self):  # pylint: disable=too-many-statements, too-many-locals
        """ Lists all the commands to be tested and their expected values.
        Expected values here refer to the expected URI that is generated
        and sent to the cluster."""

        sample_path_base = '@' + path.join(path.dirname(__file__), 'sample_json')
+        sample_yaml_base = '@' + path.join(path.dirname(__file__), 'sample_yaml')

        # The commands which don't affect or query the cluster
        # Specifically, cluster select and show-connection
@ -1108,3 +1109,52 @@ class ServiceFabricRequestTests(ScenarioTest):
            '/Resources/Secrets/some~secret~resource~name/values/secret~value~name/list_value',
            ['api-version=6.4-preview']
        )
+
+        sample_network = path.join(sample_yaml_base, 'sample_network.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_secret = path.join(sample_yaml_base, 'sample_secret.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_secret_value = path.join(sample_yaml_base, 'sample_secret_value.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_volume = path.join(sample_yaml_base, 'sample_volume.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_gateway = path.join(sample_yaml_base, 'sample_gateway.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_app = path.join(sample_yaml_base, 'sample_app.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_network),
+            'PUT',
+            '/Resources/Networks/someNetwork',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_secret),
+            'PUT',
+            '/Resources/Secrets/someSecret',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_secret_value),
+            'PUT',
+            '/Resources/Secrets/someSecret/values/v1',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_volume),
+            'PUT',
+            '/Resources/Volumes/someVolume',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_gateway),
+            'PUT',
+            '/Resources/Gateways/someGateway',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_app),
+            'PUT',
+            '/Resources/Applications/someApp',
+            ['api-version=6.4-preview']
+        )
--- a/src/sfctl/tests/sample_yaml/sample_app.yaml
+++ b/src/sfctl/tests/sample_yaml/sample_app.yaml
@ -0,0 +1,25 @@
+application:
+  schemaVersion: 1.0.0-preview2
+  name: someApp
+  properties:
+    description: someApp description.
+    services:
+    - name: someService
+      properties:
+        description: someService description.
+        osType: Windows
+        codePackages:
+        - name: someServiceCode
+          image: someImage
+          endpoints:
+          - name: someServiceListener
+            port: 80
+          resources:
+            requests:
+              cpu: 0.5
+              memoryInGB: 1
+        replicaCount: 1
+        networkRefs:
+        - name: someNetwork
+          endpointRefs:
+          - name: someServiceListener
--- a/src/sfctl/tests/sample_yaml/sample_gateway.yaml
+++ b/src/sfctl/tests/sample_yaml/sample_gateway.yaml
@ -0,0 +1,16 @@
+gateway:
+  schemaVersion: 1.0.0-preview2
+  name: someGateway
+  properties:
+    description: Gateway to connect to public internet.
+    sourceNetwork:
+        name: Open
+    destinationNetwork:
+        name: someNetwork
+    tcp:
+      - name: PublicConnection
+        port: 80
+        destination:
+            applicationName: someApp
+            serviceName: someService
+            endpointName: someListener
--- a/src/sfctl/tests/sample_yaml/sample_network.yaml
+++ b/src/sfctl/tests/sample_yaml/sample_network.yaml
@ -0,0 +1,7 @@
+network:
+  schemaVersion: 1.0.0-preview2
+  name: someNetwork
+  properties:
+    kind: Local
+    description: someNetwork description.
+    networkAddressPrefix: 8.0.0.0/16
--- a/src/sfctl/tests/sample_yaml/sample_secret.yaml
+++ b/src/sfctl/tests/sample_yaml/sample_secret.yaml
@ -0,0 +1,7 @@
+secret:
+  schemaVersion: 1.0.0-preview2
+  name: someSecret
+  properties:
+    kind: inlinedValue
+    description: Account key for azure files
+    contentType: text/plain
--- a/src/sfctl/tests/sample_yaml/sample_secret_value.yaml
+++ b/src/sfctl/tests/sample_yaml/sample_secret_value.yaml
@ -0,0 +1,5 @@
+secretValue:
+  schemaVersion: 1.0.0-preview2
+  name: someSecret/v1
+  properties:
+    value: secretValue
--- a/src/sfctl/tests/sample_yaml/sample_volume.yaml
+++ b/src/sfctl/tests/sample_yaml/sample_volume.yaml
@ -0,0 +1,10 @@
+volume:
+  schemaVersion: 1.0.0-preview2
+  name: someVolume
+  properties:
+    description: Azure Files storage volume for someApp.
+    provider: SFAzureFile
+    azureFileParameters:
+      shareName: volshare
+      accountName: someName
+      accountKey: "[reference('secrets/someSecret/values/v1')]"