Merge branch 'StorageDataPlaneAutoCommand' of https://github.com/tjprescott/azure-cli into StorageAutoCommands

# Conflicts:
#	azure-cli.pyproj
#	src/azure/cli/commands/_auto_command.py
#	src/azure/cli/commands/storage.py

azure-cli.pyproj

@@ -97,6 +97,9 @@
       <Architecture>X86</Architecture>
     </Interpreter>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="azure\cli\tests\recordings\README.rst" />
+  </ItemGroup>
   <Import Project="$(PtvsTargetsFile)" Condition="Exists($(PtvsTargetsFile))" />
   <Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="!Exists($(PtvsTargetsFile))" />
 </Project>

src/azure/cli/_output.py

@@ -104,7 +104,7 @@ class ListOutput(object): #pylint: disable=too-few-public-methods
     @staticmethod
     def _dump_line(io, line, indent):
         io.write('   ' * indent)
-        io.write(line)
+        io.write(str(line))
         io.write('\n')
 
     def _dump_object(self, io, obj, indent):

src/azure/cli/commands/_auto_command.py

@@ -6,6 +6,9 @@ from msrest.paging import Paged
 from msrest.exceptions import ClientException
 from azure.cli.parser import IncorrectUsageError
 from ..commands import COMMON_PARAMETERS
+from azure.cli._argparse import IncorrectUsageError
+from ..commands import command, description, option
+from .._logging import logger
 
 EXCLUDED_PARAMS = frozenset(['self', 'raw', 'custom_headers', 'operation_config'])
 
@@ -44,13 +47,15 @@ def _get_member(obj, path):
     Ex. a.b.c would get the property 'c' of property 'b' of the
         object a
     """
+    if not path:
+        return obj
     for segment in path.split('.'):
         obj = getattr(obj, segment)
     return obj
 
 def _make_func(client_factory, member_path, return_type_or_func, unbound_func):
     def call_client(args):
-        client = client_factory()
+        client = client_factory(args)
         ops_instance = _get_member(client, member_path)
         try:
             result = unbound_func(ops_instance, **args)
@@ -79,10 +84,13 @@ def _option_description(operation, arg):
     return ' '.join(l.split(':')[-1] for l in inspect.getdoc(operation).splitlines()
                     if l.startswith(':param') and arg + ':' in l)
 
-def build_operation(command_name, member_path, client_type, operations, command_table):
+#pylint: disable=too-many-arguments
+def build_operation(command_name, member_path, client_type, operations, #pylint: disable=dangerous-default-value
+                    paramaliases=GLOBALPARAMALIASES, extra_args=None):
     for operation, return_type_name in operations:
         opname = operation.__name__.replace('_', '-')
         func = _make_func(client_type, member_path, return_type_name, operation)
+        func = _decorate_command(' '.join([command_name, opname]), func)
 
         args = []
         try:
@@ -111,3 +119,10 @@ def build_operation(command_name, member_path, client_type, operations, command_
             'arguments': options
             }
 
+        if extra_args:
+            for arg in extra_args:
+                if len(arg) != 2:
+                    logger.warning('%s is in invalid format. Should be: (spec, description)',
+                                   (str(arg)))
+                    continue
+                func = _decorate_option(arg[0], arg[1], target=None, func=func)

src/azure/cli/commands/network.py

@@ -25,7 +25,7 @@ from ..commands import CommandTable
 
 command_table = CommandTable()
 
-def _network_client_factory():
+def _network_client_factory(*args): # pylint: disable=unused-argument
     return get_mgmt_service_client(NetworkManagementClient, NetworkManagementClientConfiguration)
 
 

src/azure/cli/commands/storage.py

@@ -2,9 +2,8 @@
 from datetime import datetime
 from os import environ
 from sys import stderr
-from six.moves import input #pylint: disable=redefined-builtin
 
-from azure.storage.blob import PublicAccess
+from azure.storage.blob import PublicAccess, BlockBlobService
 from azure.mgmt.storage import StorageManagementClient, StorageManagementClientConfiguration
 from azure.mgmt.storage.models import AccountType
 from azure.mgmt.storage.operations import StorageAccountsOperations
@@ -66,9 +65,30 @@ COMMON_PARAMETERS.update({
     }
 })
 
-def _storage_client_factory():
+def _storage_client_factory(*args): # pylint: disable=unused-argument
     return get_mgmt_service_client(StorageManagementClient, StorageManagementClientConfiguration)
 
+STORAGE_DATA_CLIENT_ARGS = [
+    ('--account-name -n <accountName>', L('the storage account name')),
+    ('--account-key -k <accountKey>', L('the storage account key')),
+    ('--connection-string -t <connectionString>', L('the storage account connection string'))
+]
+
+def _blob_data_service_factory(*args):
+    def _resolve_arg(key, envkey):
+        try:
+            value = args[0][key]
+            args[0].pop(key, None)
+        except (IndexError, KeyError):
+            value = environ.get(envkey)
+        return value
+
+    return get_data_service_client(
+        BlockBlobService,
+        _resolve_arg('account-name', 'AZURE_STORAGE_ACCOUNT'),
+        _resolve_arg('account-key', 'AZURE_STORAGE_ACCOUNT_KEY'),
+        _resolve_arg('connection-string', 'AZURE_STORAGE_CONNECTION_STRING'))
+
 # ACCOUNT COMMANDS
 
 build_operation('storage account',
@@ -203,6 +223,16 @@ def set_account(args, unexpected): #pylint: disable=unused-argument
 
 # CONTAINER COMMANDS
 
+build_operation('storage container', None, _blob_data_service_factory,
+                [
+                    (BlockBlobService.list_containers, '[Container]'),
+                    (BlockBlobService.delete_container, 'None'),
+                    (BlockBlobService.get_container_properties, '[ContainerProperties]'),
+                    (BlockBlobService.create_container, 'None')
+                ],
+                extra_args=STORAGE_DATA_CLIENT_ARGS)
+
+# TODO: update this once enums are supported in commands first-class (task #115175885)
 public_access_types = {'none': None,
                        'blob': PublicAccess.Blob,
                        'container': PublicAccess.Container}
@@ -272,45 +302,12 @@ def delete_container(args):
 @option('--snapshot <datetime>', L('UTC datetime value which specifies a snapshot'))
 @option('--timeout <seconds>')
 def exists_container(args, unexpected): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
+    bbs = _blob_data_service_factory(args)
     return str(bbs.exists(
         container_name=args.get('container-name'),
         snapshot=_parse_datetime(args, 'snapshot'),
         timeout=_parse_int(args, 'timeout')))
 
-@command_table.command('storage container list')
-@command_table.description(L('List storage containers.'))
-@command_table.option(**COMMON_PARAMETERS['account-name'])
-@command_table.option(**COMMON_PARAMETERS['account_key'])
-@command_table.option(**COMMON_PARAMETERS['connection-string'])
-@command_table.option('--prefix -p', help=L('container name prefix to filter by'))
-@command_table.option('--num-results <num>')
-@command_table.option('--include-metadata')
-@command_table.option('--marker <marker>', L('continuation token for enumerating additional results'))
-@command_table.option(**COMMON_PARAMETERS['timeout'])
-def list_containers(args):
-    bbs = _get_blob_service_client(args)
-    return bbs.list_containers(
-        prefix=args.get('prefix'),
-        num_results=_parse_int(args, 'num-results'),
-        include_metadata=True if args.get('include-metadata') else False,
-        marker=args.get('marker'),
-        timeout=_parse_int(args, 'timeout'))
-
-@command_table.command('storage container show')
-@command_table.description(L('Show details of a storage container'))
-@command_table.option(**COMMON_PARAMETERS['container-name'])
-@command_table.option(**COMMON_PARAMETERS['account-name'])
-@command_table.option(**COMMON_PARAMETERS['account_key'])
-@command_table.option(**COMMON_PARAMETERS['connection-string'])
-@command_table.option('--lease-id <id>', L('delete only if lease is ID active and matches'))
-@command_table.option(**COMMON_PARAMETERS['timeout'])
-def show_container(args):
-    bbs = _get_blob_service_client(args)
-    return bbs.get_container_properties(
-        container_name=args.get('container-name'),
-        lease_id=args.get('lease-id'),
-        timeout=_parse_int(args, 'timeout'))
 
 lease_duration_values = {'min':15, 'max':60, 'infinite':-1}
 lease_duration_values_string = 'Between {} and {} seconds. ({} for infinite)'.format(
@@ -318,6 +315,14 @@ lease_duration_values_string = 'Between {} and {} seconds. ({} for infinite)'.fo
     lease_duration_values['max'],
     lease_duration_values['infinite'])
 
+build_operation('storage container lease', None, _blob_data_service_factory,
+                [
+                    (BlockBlobService.renew_container_lease, 'None'),
+                    (BlockBlobService.release_container_lease, 'None'),
+                    (BlockBlobService.change_container_lease, 'LeaseId')
+                ],
+                extra_args=STORAGE_DATA_CLIENT_ARGS)
+
 @command_table.command('storage container lease acquire')
 @command_table.description(L('Acquire a lock on a container for delete operations.'))
 @command_table.option(**COMMON_PARAMETERS['container-name'])
@@ -333,7 +338,7 @@ lease_duration_values_string = 'Between {} and {} seconds. ({} for infinite)'.fo
     'modified since supplied UTC datetime'))
 @command_table.option(**COMMON_PARAMETERS['timeout'])
 def acquire_container_lease(args): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
+    bbs = _blob_data_service_factory(args)
     try:
         lease_duration = int(args.get('lease-duration'))
     except ValueError:
@@ -347,71 +352,6 @@ def acquire_container_lease(args): #pylint: disable=unused-argument
         if_unmodified_since=_parse_int(args, 'if-unmodified-since'),
         timeout=True if args.get('timeout') else False)
 
-@command_table.command('storage container lease renew')
-@command_table.description(L('Renew a lock on a container for delete operations.'))
-@command_table.option(**COMMON_PARAMETERS['container-name'])
-@command_table.option('--lease-id --lid <id>', L('lease id to renew in GUID format'), required=True)
-@command_table.option('--account-name -n <accountName>', L('the storage account name'))
-@command_table.option('--account-key -k <accountKey>', L('the storage account key'))
-@command_table.option('--connection-string -t <connectionString>', L('the storage connection string'))
-@command_table.option('--if-modified-since <dateTime>', L('delete only if container modified since ' + \
-    'supplied UTC datetime'))
-@option('--in-unmodified-since <dateTime>', L('delete only if container has not been modified ' + \
-    'since supplied UTC datetime'))
-@option('--timeout <seconds>')
-def renew_container_lease(args, unexpected): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
-    return bbs.renew_container_lease(
-        container_name=args.get('container-name'),
-        lease_id=args.get('lease-id'),
-        if_modified_since=_parse_int(args, 'if-modified-since'),
-        if_unmodified_since=_parse_int(args, 'if-unmodified-since'),
-        timeout=True if args.get('timeout') else False)
-
-@command('storage container lease release')
-@description(L('Release a lock on a container for delete operations.'))
-@option('--container-name -c <containerName>', L('the name of the container'), required=True)
-@option('--lease-id --lid <id>', L('lease id in GUID format to release'), required=True)
-@option('--account-name -n <accountName>', L('the storage account name'))
-@option('--account-key -k <accountKey>', L('the storage account key'))
-@option('--connection-string -t <connectionString>', L('the storage connection string'))
-@option('--if-modified-since <dateTime>', L('delete only if container modified since ' + \
-    'supplied UTC datetime'))
-@option('--in-unmodified-since <dateTime>', L('delete only if container has not been modified ' + \
-    'since supplied UTC datetime'))
-@option('--timeout <seconds>')
-def release_container_lease(args, unexpected): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
-    bbs.release_container_lease(
-        container_name=args.get('container-name'),
-        lease_id=args.get('lease-id'),
-        if_modified_since=_parse_int(args, 'if-modified-since'),
-        if_unmodified_since=_parse_int(args, 'if-unmodified-since'),
-        timeout=True if args.get('timeout') else False)
-
-@command('storage container lease change')
-@description(L('Change the lease id for a container lease.'))
-@option('--container-name -c <containerName>', L('the name of the container'), required=True)
-@option('--lease-id --lid <leaseId>', L('the lease id to change'), required=True)
-@option('--proposed-lease-id --plid <id>', L('proposed lease id in GUID format'))
-@option('--account-name -n <accountName>', L('the storage account name'))
-@option('--account-key -k <accountKey>', L('the storage account key'))
-@option('--connection-string -t <connectionString>', L('the storage connection string'))
-@option('--if-modified-since <dateTime>', L('delete only if container modified since ' + \
-    'supplied UTC datetime'))
-@option('--in-unmodified-since <dateTime>', L('delete only if container has not been modified ' + \
-    'since supplied UTC datetime'))
-@option('--timeout <seconds>')
-def change_container_lease(args, unexpected): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
-    return bbs.change_container_lease(
-        container_name=args.get('container-name'),
-        lease_id=args.get('lease-id'),
-        proposed_lease_id=args.get('proposed-lease-id'),
-        if_modified_since=_parse_int(args, 'if-modified-since'),
-        if_unmodified_since=_parse_int(args, 'if-unmodified-since'),
-        timeout=True if args.get('timeout') else False)
-
 @command('storage container lease break')
 @description(L('Break a lock on a container for delete operations.'))
 @option('--container-name -c <containerName>', L('the name of the container'), required=True)
@@ -426,12 +366,11 @@ def change_container_lease(args, unexpected): #pylint: disable=unused-argument
     'since supplied UTC datetime'))
 @option('--timeout <seconds>')
 def break_container_lease(args, unexpected): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
+    bbs = _blob_data_service_factory(args)
     try:
         lease_break_period = int(args.get('lease-break-period'))
     except ValueError:
         raise ValueError('lease-break-period must be: {}'.format(lease_duration_values_string))
-
     bbs.break_container_lease(
         container_name=args.get('container-name'),
         lease_break_period=lease_break_period,
@@ -440,7 +379,15 @@ def break_container_lease(args, unexpected): #pylint: disable=unused-argument
         timeout=True if args.get('timeout') else False)
 
 # BLOB COMMANDS
-# TODO: Evaluate for removing hand-authored commands in favor of auto-commands (task ##115068835)
+
+build_operation('storage blob', None, _blob_data_service_factory,
+                [
+                    (BlockBlobService.list_blobs, '[Blob]'),
+                    (BlockBlobService.delete_blob, 'None'),
+                    (BlockBlobService.exists, 'Boolean'),
+                    (BlockBlobService.get_blob_properties, 'BlobProperties')
+                ],
+                extra_args=STORAGE_DATA_CLIENT_ARGS)
 
 @command_table.command('storage blob upload-block-blob')
 @command_table.description(L('Upload a block blob to a container.'))
@@ -461,8 +408,15 @@ def break_container_lease(args, unexpected): #pylint: disable=unused-argument
 @command_table.option('--content.cache-control -cscc')
 def create_block_blob(args):
     from azure.storage.blob import ContentSettings
-    bbs = _get_blob_service_client(args)
-    public_access = args.get('container.public-access')
+    bbs = _blob_data_service_factory(args)
+    try:
+        public_access = public_access_types[args.get('public-access')] \
+                                            if args.get('public-access') \
+                                            else None
+    except KeyError:
+        raise IncorrectUsageError(L('container.public-access must be: {}'
+                                    .format(public_access_string)))
+
     bbs.create_container(args.get('container-name'), public_access=public_access)
 
     return bbs.create_blob_from_path(
@@ -478,80 +432,77 @@ def create_block_blob(args):
                                          cache_control=args.get('content.cache-control'))
         )
 
+#@command('storage blob list')
+#@command(L('List all blobs in a container.'))
+#@option('--container-name -c <containerName>', L('the name of the container'), required=True)
+#@option('--account-name -n <accountName>', L('the storage account name'))
+#@option('--account-key -k <accountKey>', L('the storage account key'))
+#@option('--connection-string -t <connectionString>', L('the storage connection string'))
+#@option('--prefix -p <prefix>', L('blob name prefix to filter by'))
+#@option('--num-results <num>')
+#@option('--include <stuff>', L('specifies one or more additional datasets to include '\
+#    + 'in the response. Unsupported this release'))
+#@option('--delimiter <value>', L('Unsupported this release'))
+#@option('--marker <marker>', L('continuation token for enumerating additional results'))
+#@option('--timeout <seconds>')
+#def list_blobs(args, unexpected): #pylint: disable=unused-argument
+#    bbs = _blob_data_service_factory(args)
+#    blobs = bbs.list_blobs(
+#        container_name=args.get('container-name'),
+#        prefix=args.get('prefix'),
+#        num_results=_parse_int(args, 'num-results'),
+#        include=None,
+#        delimiter=None,
+#        marker=args.get('marker'),
+#        timeout=_parse_int(args, 'timeout'))
+#    return list(blobs.items)
 
-@option('--num-results <num>')
-@option('--include <stuff>', L('specifies one or more additional datasets to include '\
-    + 'in the response. Unsupported this release'))
-@option('--delimiter <value>', L('Unsupported this release'))
-@option('--marker <marker>', L('continuation token for enumerating additional results'))
-@option('--timeout <seconds>')
-def list_blobs(args, unexpected): #pylint: disable=unused-argument
-@command_table.command('storage blob list')
-@command_table.description(L('List all blobs in a container.'))
-@command_table.option(**COMMON_PARAMETERS['container-name'])
-@command_table.option(**COMMON_PARAMETERS['account-name'])
-@command_table.option(**COMMON_PARAMETERS['account_key'])
-@command_table.option(**COMMON_PARAMETERS['connection-string'])
-@command_table.option('--prefix -p', help=L('blob name prefix to filter by'))
-def list_blobs(args):
-    bbs = _get_blob_service_client(args)
-    blobs = bbs.list_blobs(
-        container_name=args.get('container-name'),
-        prefix=args.get('prefix'),
-        num_results=_parse_int(args, 'num-results'),
-        include=None,
-        delimiter=None,
-        marker=args.get('marker'),
-        timeout=_parse_int(args, 'timeout'))
-    return list(blobs.items)
+#@command('storage blob delete')
+#@description(L('Delete a blob from a container.'))
+#@option('--container-name -c <containerName>', L('the name of the container'), required=True)
+#@option('--blob-name -b <blobName>', L('the name of the blob'), required=True)
+#@option('--account-name -n <accountName>', L('the storage account name'))
+#@option('--account-key -k <accountKey>', L('the storage account key'))
+#@option('--connection-string -t <connectionString>', L('the storage connection string'))
+#def delete_blob(args, unexpected): #pylint: disable=unused-argument
+#    bbs = _blob_data_service_factory(args)
+#    return bbs.delete_blob(args.get('container-name'), args.get('blob-name'))
 
-def delete_blob(args, unexpected): #pylint: disable=unused-argument
-@command_table.command('storage blob delete')
-@command_table.description(L('Delete a blob from a container.'))
-@command_table.option(**COMMON_PARAMETERS['container-name'])
-@command_table.option(**COMMON_PARAMETERS['blob-name'])
-def delete_blob(args):
-    bbs = _get_blob_service_client(args)
-    return bbs.delete_blob(args.get('container-name'), args.get('blob-name'))
+#@command('storage blob exists')
+#@description(L('Check if a blob exists.'))
+#@option('--container-name -c <containerName>', L('the name of the container'), required=True)
+#@option('--blob-name -b <blobName>', L('the name of the blob'), required=True)
+#@option('--account-name -n <accountName>', L('the storage account name'))
+#@option('--account-key -k <accountKey>', L('the storage account key'))
+#@option('--connection-string -t <connectionString>', L('the storage connection string'))
+#@option('--snapshot <datetime>', L('UTC datetime value which specifies a snapshot'))
+#@option('--timeout <seconds>')
+#def exists_blob(args, unexpected): #pylint: disable=unused-argument
+#    bbs = _blob_data_service_factory(args)
+#    return str(bbs.exists(
+#        container_name=args.get('container-name'),
+#        blob_name=args.get('blob-name'),
+#        snapshot=_parse_datetime(args, 'snapshot'),
+#        timeout=_parse_int(args, 'timeout')))
 
-@command('storage blob exists')
-@description(L('Check if a blob exists.'))
-@option('--container-name -c <containerName>', L('the name of the container'), required=True)
-@option('--blob-name -b <blobName>', L('the name of the blob'), required=True)
-@option('--account-name -n <accountName>', L('the storage account name'))
-@option('--account-key -k <accountKey>', L('the storage account key'))
-@option('--connection-string -t <connectionString>', L('the storage connection string'))
-@option('--snapshot <datetime>', L('UTC datetime value which specifies a snapshot'))
-@option('--timeout <seconds>')
-def exists_blob(args, unexpected): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
-    return str(bbs.exists(
-        container_name=args.get('container-name'),
-        blob_name=args.get('blob-name'),
-        snapshot=_parse_datetime(args, 'snapshot'),
-        timeout=_parse_int(args, 'timeout')))
+#@command('storage blob show')
+#@description(L('Show properties of the specified blob.'))
+#@option('--container-name -c <containerName>', L('the name of the container'), required=True)
+#@option('--blob-name -bn <name>', L('the name of the blob'), required=True)
+#@option('--account-name -n <accountName>', L('the storage account name'))
+#@option('--account-key -k <accountKey>', L('the storage account key'))
+#@option('--connection-string -t <connectionString>', L('the storage connection string'))
+#def show_blob(args, unexpected): #pylint: disable=unused-argument
+#    bbs = _blob_data_service_factory(args)
+#    return bbs.get_blob_properties(args.get('container-name'), args.get('blob-name'))
 
-@command('storage blob show')
-@description(L('Show properties of the specified blob.'))
-@option('--container-name -c <containerName>', L('the name of the container'), required=True)
-@option('--blob-name -bn <name>', L('the name of the blob'), required=True)
-@option('--account-name -n <accountName>', L('the storage account name'))
-@option('--account-key -k <accountKey>', L('the storage account key'))
-@option('--connection-string -t <connectionString>', L('the storage connection string'))
-def show_blob(args, unexpected): #pylint: disable=unused-argument
-    bbs = _get_blob_service_client(args)
-    return bbs.get_blob_properties(args.get('container-name'), args.get('blob-name'))
-
-@option('--account-name -n <accountName>', L('the storage account name'))
-@option('--account-key -k <accountKey>', L('the storage account key'))
-@option('--connection-string -t <connectionString>', L('the storage connection string'))
 @command_table.command('storage blob download')
 @command_table.description(L('Download the specified blob.'))
 @command_table.option(**COMMON_PARAMETERS['container-name'])
 @command_table.option(**COMMON_PARAMETERS['blob-name'])
 @command_table.option('--download-to -dt', help=L('the file path to download to'), required=True)
 def download_blob(args):
-    bbs = _get_blob_service_client(args)
+    bbs = _blob_data_service_factory(args)
 
     # show dot indicator of download progress (one for every 10%)
     bbs.get_blob_to_path(args.get('container-name'),
@@ -727,13 +678,6 @@ def storage_file_delete(args, unexpected): #pylint: disable=unused-argument
 
 # HELPER METHODS
 
-def _get_blob_service_client(args):
-    from azure.storage.blob import BlockBlobService
-    return get_data_service_client(BlockBlobService,
-                                   args.get('storage-account', None),
-                                   args.get('storage-account-key', None),
-                                   args.get('connection-string', None))
-
 def _get_file_service_client(args):
     from azure.storage.file import FileService
     return get_data_service_client(FileService,

src/azure/cli/commands/vm.py

@@ -14,7 +14,7 @@ from ._command_creation import get_mgmt_service_client
 from ..commands._auto_command import build_operation, LongRunningOperation
 from ..commands import CommandTable
 
-def _compute_client_factory():
+def _compute_client_factory(*args): # pylint: disable=unused-argument
     return get_mgmt_service_client(ComputeManagementClient, ComputeManagementClientConfiguration)
 
 command_table = CommandTable()