Refactor and unify AAD settings across commands

- All KeyVault AAD endpoints to be specified

config_templates/credentials.json

@@ -10,20 +10,29 @@
                 "rsa_private_key_pem": "",
                 "x509_cert_sha1_thumbprint": "",
                 "user": "",
-                "password": ""
+                "password": "",
+                "endpoint": "https://vault.azure.net",
+                "token_cache": {
+                    "enabled": true,
+                    "filename": ""
+                }
             }
         },
         "management": {
             "subscription_id": "",
             "aad": {
                 "directory_id": "",
+                "application_id": "",
+                "auth_key": "",
+                "rsa_private_key_pem": "",
+                "x509_cert_sha1_thumbprint": "",
                 "user": "",
-                "password": ""
-            },
-            "endpoint": "https://management.core.windows.net/",
-            "token_cache": {
-                "enabled": true,
-                "filename": ""
+                "password": "",
+                "endpoint": "https://management.core.windows.net/",
+                "token_cache": {
+                    "enabled": true,
+                    "filename": ""
+                }
             }
         },
         "batch": {

convoy/aad.py

@@ -0,0 +1,251 @@
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# compat imports
+from __future__ import (
+    absolute_import, division, print_function, unicode_literals
+)
+from builtins import (  # noqa
+    bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
+    next, oct, open, pow, round, super, filter, map, zip)
+# stdlib imports
+import datetime
+import dateutil.parser
+import json
+import logging
+try:
+    import pathlib2 as pathlib
+except ImportError:
+    import pathlib
+# non-stdlib imports
+import adal
+import azure.common.credentials
+import msrest.authentication
+import msrestazure.azure_exceptions
+# local imports
+from . import util
+
+# create logger
+logger = logging.getLogger(__name__)
+util.setup_logger(logger)
+# global defines
+_LOGIN_AUTH_URI = 'https://login.microsoftonline.com'
+_CLIENT_ID = '04b07795-8ddb-461a-bbee-02f9e1bf7b46'  # xplat-cli
+
+
+class DeviceCodeAuthentication(msrest.authentication.Authentication):
+    """Device Code Authentication session handler"""
+    def __init__(self, context, resource, client_id, token_cache_file):
+        """Ctor for DeviceCodeAuthentication
+        :param DeviceCodeAuthentication self: this
+        :param object context: context
+        :param str resource: resource
+        :param str client_id: client id
+        :param str token_Cache_file: token cache file
+        """
+        self._context = context
+        self._resource = resource
+        self._client_id = client_id
+        self._token_cache_file = token_cache_file
+        self._token = None
+
+    @property
+    def token(self):
+        """Retrieve signed token
+        :param DeviceCodeAuthentication self: this
+        """
+        return self._token
+
+    @token.setter
+    def token(self, value):
+        """Set signed token
+        :param DeviceCodeAuthentication self: this
+        :param str value: token value
+        """
+        self._token = value
+
+    def signed_session(self):
+        """Get a signed session for requests.
+        Usually called by the Azure SDKs for you to authenticate queries.
+        :param DeviceCodeAuthentication self: this
+        :rtype: requests.Session
+        :return: request session with signed header
+        """
+        session = super(DeviceCodeAuthentication, self).signed_session()
+        # try to get cached token
+        if self._token is None and util.is_not_empty(self._token_cache_file):
+            try:
+                with open(self._token_cache_file, 'r') as fd:
+                    self._token = json.load(fd)
+            except OSError:
+                pass
+            except Exception:
+                logger.error(
+                    'Error attempting read of token cache: {}'.format(
+                        self._token_cache_file))
+        # get token
+        try:
+            cache_token = True
+            if self._token is None:
+                # get token through selected method
+                code = self._context.acquire_user_code(
+                    resource=self._resource,
+                    client_id=self._client_id,
+                )
+                logger.info(
+                    'Please follow the instructions below. The requesting '
+                    'application will be: Microsoft Azure Cross-platform '
+                    'Command Line Interface')
+                logger.info(code['message'])
+                self._token = self._context.acquire_token_with_device_code(
+                    resource=self._resource,
+                    user_code_info=code,
+                    client_id=self._client_id,
+                )
+            else:
+                # check for expiry time
+                expiry = dateutil.parser.parse(self._token['expiresOn'])
+                if (datetime.datetime.now() +
+                        datetime.timedelta(minutes=5) >= expiry):
+                    # attempt token refresh
+                    logger.debug('Refreshing token expiring on: {}'.format(
+                        expiry))
+                    self._token = self._context.\
+                        acquire_token_with_refresh_token(
+                            refresh_token=self._token['refreshToken'],
+                            client_id=self._client_id,
+                            resource=self._resource,
+                        )
+                else:
+                    cache_token = False
+            # set session authorization header
+            session.headers['Authorization'] = '{} {}'.format(
+                self._token['tokenType'], self._token['accessToken'])
+            # cache token
+            if cache_token and util.is_not_empty(self._token_cache_file):
+                logger.debug('storing token to local cache: {}'.format(
+                    self._token_cache_file))
+                with open(self._token_cache_file, 'w') as fd:
+                    json.dump(self._token, fd, indent=4, sort_keys=False)
+        except adal.AdalError as err:
+            if (hasattr(err, 'error_response') and
+                    'error_description' in err.error_response and
+                    'AADSTS70008:' in err.error_response['error_description']):
+                logger.error(
+                    'Credentials have expired due to inactivity. Please '
+                    'retry your command.')
+            # clear token cache file due to expiration
+            if util.is_not_empty(self._token_cache_file):
+                try:
+                    pathlib.Path(self._token_cache_file).unlink()
+                    logger.debug('invalidated local token cache: {}'.format(
+                        self._token_cache_file))
+                except OSError:
+                    pass
+            raise
+        return session
+
+
+def create_aad_credentials(ctx, aad_settings):
+    # type: (CliContext, settings.AADSettings) ->
+    #       azure.common.credentials.ServicePrincipalCredentials or
+    #       azure.common.credentials.UserPassCredentials
+    """Create Azure Active Directory credentials
+    :param CliContext ctx: Cli Context
+    :param settings.AADSettings aad_settings: AAD settings
+    :rtype: azure.common.credentials.ServicePrincipalCredentials or
+        azure.common.credentials.UserPassCredentials
+    :return: aad credentials object
+    """
+    # from aad parameters
+    aad_directory_id = ctx.aad_directory_id or aad_settings.directory_id
+    aad_application_id = ctx.aad_application_id or aad_settings.application_id
+    aad_auth_key = ctx.aad_auth_key or aad_settings.auth_key
+    aad_user = ctx.aad_user or aad_settings.user
+    aad_password = ctx.aad_password or aad_settings.password
+    aad_cert_private_key = (
+        ctx.aad_cert_private_key or aad_settings.rsa_private_key_pem
+    )
+    aad_cert_thumbprint = (
+        ctx.aad_cert_thumbprint or aad_settings.x509_cert_sha1_thumbprint
+    )
+    endpoint = ctx.aad_endpoint or aad_settings.endpoint
+    token_cache_file = aad_settings.token_cache_file
+    # check for aad parameter validity
+    if ((aad_directory_id is None and aad_application_id is None and
+         aad_auth_key is None and aad_user is None and
+         aad_password is None and aad_cert_private_key is None and
+         aad_cert_thumbprint is None) or endpoint is None):
+        return None
+    # create credential object
+    if (util.is_not_empty(aad_application_id) and
+            util.is_not_empty(aad_cert_private_key)):
+        if util.is_not_empty(aad_auth_key):
+            raise ValueError('cannot specify both cert auth and auth key')
+        if util.is_not_empty(aad_password):
+            raise ValueError('cannot specify both cert auth and password')
+        logger.debug('authenticating with certificate')
+        context = adal.AuthenticationContext(
+            '{}/{}'.format(_LOGIN_AUTH_URI, aad_directory_id))
+        return msrestazure.azure_active_directory.AdalAuthentication(
+            lambda: context.acquire_token_with_client_certificate(
+                endpoint,
+                aad_application_id,
+                util.decode_string(open(aad_cert_private_key, 'rb').read()),
+                aad_cert_thumbprint
+            )
+        )
+    elif util.is_not_empty(aad_auth_key):
+        if util.is_not_empty(aad_password):
+            raise ValueError(
+                'Cannot specify both an AAD Service Principal and User')
+        logger.debug('authenticating with auth key')
+        return azure.common.credentials.ServicePrincipalCredentials(
+            aad_application_id,
+            aad_auth_key,
+            tenant=aad_directory_id,
+            resource=endpoint,
+        )
+    elif util.is_not_empty(aad_password):
+        logger.debug('authenticating with username and password')
+        try:
+            return azure.common.credentials.UserPassCredentials(
+                username=aad_user,
+                password=aad_password,
+                resource=endpoint,
+            )
+        except msrest.exceptions.AuthenticationError as e:
+            if 'AADSTS50079' in e.args[0]:
+                raise RuntimeError('{} {}'.format(
+                    e.args[0][2:],
+                    'Do not pass an AAD password and try again.'))
+    else:
+        logger.debug('authenticating with device code')
+        return DeviceCodeAuthentication(
+            context=adal.AuthenticationContext(
+                '{}/{}'.format(_LOGIN_AUTH_URI, aad_directory_id)),
+            resource=endpoint,
+            client_id=_CLIENT_ID,
+            token_cache_file=token_cache_file,
+        )

convoy/fleet.py

@@ -244,23 +244,7 @@ def create_keyvault_client(ctx, config):
     :return: key vault client
     """
     kv = settings.credentials_keyvault(config)
-    aad_directory_id = ctx.aad_directory_id or kv.aad_directory_id
-    aad_application_id = ctx.aad_application_id or kv.aad_application_id
-    aad_auth_key = ctx.aad_auth_key or kv.aad_auth_key
-    aad_user = ctx.aad_user or kv.aad_user
-    aad_password = ctx.aad_password or kv.aad_password
-    aad_cert_private_key = ctx.aad_cert_private_key or kv.aad_cert_private_key
-    aad_cert_thumbprint = ctx.aad_cert_thumbprint or kv.aad_cert_thumbprint
-    # check if no keyvault/aad params were specified at all
-    if (aad_directory_id is None and aad_application_id is None and
-            aad_auth_key is None and aad_user is None and
-            aad_password is None and aad_cert_private_key is None and
-            aad_cert_thumbprint is None):
-        return None
-    else:
-        return keyvault.create_client(
-            aad_directory_id, aad_application_id, aad_auth_key, aad_user,
-            aad_password, aad_cert_private_key, aad_cert_thumbprint)
+    return keyvault.create_client(ctx, kv.aad)
 
 
 def create_remotefs_clients(ctx, config):
@@ -278,18 +262,8 @@ def create_remotefs_clients(ctx, config):
         azure.mgmt.network.NetworkManagementClient)
     """
     mgmt = settings.credentials_management(config)
-    aad_directory_id = ctx.aad_directory_id or mgmt.aad_directory_id
-    aad_user = ctx.aad_user or mgmt.aad_user
-    aad_password = ctx.aad_password or mgmt.aad_password
     subscription_id = ctx.subscription_id or mgmt.subscription_id
-    # check if no management/aad params were specified at all
-    if (aad_directory_id is None and aad_user is None and
-            subscription_id is None):
-        return (None, None, None)
-    else:
-        return remotefs.create_clients(
-            subscription_id, aad_directory_id, aad_user, aad_password,
-            mgmt.endpoint, mgmt.token_cache_file)
+    return remotefs.create_clients(ctx, mgmt.aad, subscription_id)
 
 
 def initialize(config, remotefs_context=False):

convoy/keyvault.py

@@ -34,11 +34,10 @@ import json
 import logging
 import zlib
 # non-stdlib imports
-import adal
 import azure.common.credentials
 import azure.keyvault
-import msrestazure.azure_active_directory
 # local imports
+from . import aad
 from . import settings
 from . import util
 
@@ -46,85 +45,22 @@ from . import util
 logger = logging.getLogger(__name__)
 util.setup_logger(logger)
 # global defines
-_KEYVAULT_RESOURCE = 'https://vault.azure.net'
 _SECRET_ENCODED_FORMAT_KEY = 'format'
 _SECRET_ENCODED_FORMAT_VALUE = 'zlib+base64'
 
 
-def _create_aad_credentials(
-        aad_directory_id, aad_application_id, aad_auth_key, aad_user,
-        aad_password, aad_cert_private_key, aad_cert_thumbprint):
-    # type: (str, str, str, str, str, str, str) ->
-    #       azure.common.credentials.ServicePrincipalCredentials or
-    #       azure.common.credentials.UserPassCredentials
-    """Create Azure Active Directory credentials
-    :param str aad_directory_id: aad directory/tenant id
-    :param str aad_application_id: aad application/client id
-    :param str aad_auth_key: aad auth key
-    :param str aad_user: aad user
-    :param str aad_password: aad_password
-    :param str aad_cert_private_key: path to rsa private key
-    :param str aad_cert_thumbprint: sha1 thumbprint
-    :rtype: azure.common.credentials.ServicePrincipalCredentials or
-            azure.common.credentials.UserPassCredentials
-    :return: aad credentials object
-    """
-    if aad_application_id is not None and aad_cert_private_key is not None:
-        if aad_auth_key is not None:
-            raise ValueError('cannot specify both cert auth and auth key')
-        if aad_password is not None:
-            raise ValueError('cannot specify both cert auth and password')
-        context = adal.AuthenticationContext(
-            'https://login.microsoftonline.com/{}'.format(aad_directory_id))
-        return msrestazure.azure_active_directory.AdalAuthentication(
-            lambda: context.acquire_token_with_client_certificate(
-                _KEYVAULT_RESOURCE,
-                aad_application_id,
-                util.decode_string(open(aad_cert_private_key, 'rb').read()),
-                aad_cert_thumbprint
-            )
-        )
-    elif aad_auth_key is not None:
-        if aad_password is not None:
-            raise ValueError(
-                'cannot specify both an AAD Service Principal and User')
-        return azure.common.credentials.ServicePrincipalCredentials(
-            aad_application_id,
-            aad_auth_key,
-            tenant=aad_directory_id,
-            resource=_KEYVAULT_RESOURCE,
-        )
-    elif aad_password is not None:
-        return azure.common.credentials.UserPassCredentials(
-            username=aad_user,
-            password=aad_password,
-            resource=_KEYVAULT_RESOURCE,
-        )
-    else:
-        raise ValueError(
-            'AAD Service Principal, User or Certificate not specified')
-
-
-def create_client(
-        aad_directory_id, aad_application_id, aad_auth_key, aad_user,
-        aad_password, aad_cert_private_key, aad_cert_thumbprint):
-    # type: (str, str, str, str, str, str, str) ->
+def create_client(ctx, kv_aad):
+    # type: (CliContext, settings.AADSettings) ->
     #       azure.keyvault.KeyVaultClient
     """Create KeyVault client
-    :param str aad_directory_id: aad directory/tenant id
-    :param str aad_application_id: aad application/client id
-    :param str aad_auth_key: aad auth key
-    :param str aad_user: aad user
-    :param str aad_password: aad_password
-    :param str aad_cert_private_key: path to rsa private key
-    :param str aad_cert_thumbprint: sha1 thumbprint
+    :param CliContext ctx: Cli Context
+    :param settings.AADSettings kv_aad: AAD settings
     :rtype: azure.keyvault.KeyVaultClient
     :return: keyvault client
     """
-    credentials = _create_aad_credentials(
-        aad_directory_id, aad_application_id, aad_auth_key, aad_user,
-        aad_password, aad_cert_private_key, aad_cert_thumbprint)
-    return azure.keyvault.KeyVaultClient(credentials)
+    return azure.keyvault.KeyVaultClient(
+        aad.create_aad_credentials(ctx, kv_aad)
+    )
 
 
 def fetch_credentials_json(

convoy/remotefs.py

@@ -30,8 +30,6 @@ from builtins import (  # noqa
     bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
     next, oct, open, pow, round, super, filter, map, zip)
 # stdlib imports
-import datetime
-import dateutil.parser
 import json
 import logging
 import os
@@ -40,7 +38,6 @@ try:
 except ImportError:
     import pathlib
 # non-stdlib imports
-import adal
 import azure.common.credentials
 import azure.mgmt.compute
 import azure.mgmt.compute.models as computemodels
@@ -48,9 +45,9 @@ import azure.mgmt.network
 import azure.mgmt.network.models as networkmodels
 import azure.mgmt.resource
 import azure.mgmt.resource.resources.models as rgmodels
-import msrest.authentication
 import msrestazure.azure_exceptions
 # local imports
+from . import aad
 from . import crypto
 from . import settings
 from . import storage
@@ -60,164 +57,27 @@ from . import util
 logger = logging.getLogger(__name__)
 util.setup_logger(logger)
 # global defines
-_CLIENT_ID = '04b07795-8ddb-461a-bbee-02f9e1bf7b46'  # xplat-cli
 _SSH_KEY_PREFIX = 'id_rsa_shipyard_remotefs'
 
 
-class DeviceCodeAuthentication(msrest.authentication.Authentication):
-    def __init__(self, context, resource, client_id, token_cache_file):
-        self._context = context
-        self._resource = resource
-        self._client_id = client_id
-        self._token_cache_file = token_cache_file
-        self._token = None
-
-    @property
-    def token(self):
-        return self._token
-
-    @token.setter
-    def token(self, value):
-        self._token = value
-
-    def signed_session(self):
-        """Get a signed session for requests.
-        Usually called by the Azure SDKs for you to authenticate queries.
-        :rtype: requests.Session
-        """
-        session = super(DeviceCodeAuthentication, self).signed_session()
-        # try to get cached token
-        if self._token is None and util.is_not_empty(self._token_cache_file):
-            try:
-                with open(self._token_cache_file, 'r') as fd:
-                    self._token = json.load(fd)
-            except OSError:
-                pass
-            except Exception:
-                logger.error(
-                    'Error attempting read of token cache: {}'.format(
-                        self._token_cache_file))
-        # get token
-        try:
-            cache_token = True
-            if self._token is None:
-                # get token through selected method
-                code = self._context.acquire_user_code(
-                    resource=self._resource,
-                    client_id=self._client_id,
-                )
-                logger.info(
-                    'Please follow the instructions below. The requesting '
-                    'application will be: Microsoft Azure Cross-platform '
-                    'Command Line Interface')
-                logger.info(code['message'])
-                self._token = self._context.acquire_token_with_device_code(
-                    resource=self._resource,
-                    user_code_info=code,
-                    client_id=self._client_id,
-                )
-            else:
-                # check for expiry time
-                expiry = dateutil.parser.parse(self._token['expiresOn'])
-                if (datetime.datetime.now() +
-                        datetime.timedelta(minutes=5) >= expiry):
-                    # attempt token refresh
-                    logger.debug('Refreshing token expiring on: {}'.format(
-                        expiry))
-                    self._token = self._context.\
-                        acquire_token_with_refresh_token(
-                            refresh_token=self._token['refreshToken'],
-                            client_id=self._client_id,
-                            resource=self._resource,
-                        )
-                else:
-                    cache_token = False
-            # set session authorization header
-            session.headers['Authorization'] = '{} {}'.format(
-                self._token['tokenType'], self._token['accessToken'])
-            # cache token
-            if cache_token and util.is_not_empty(self._token_cache_file):
-                logger.debug('storing token to local cache: {}'.format(
-                    self._token_cache_file))
-                with open(self._token_cache_file, 'w') as fd:
-                    json.dump(self._token, fd, indent=4, sort_keys=False)
-        except adal.AdalError as err:
-            if (hasattr(err, 'error_response') and
-                    'error_description' in err.error_response and
-                    'AADSTS70008:' in err.error_response['error_description']):
-                logger.error(
-                    'Credentials have expired due to inactivity. Please '
-                    'retry your command.')
-            # clear token cache file due to expiration
-            if util.is_not_empty(self._token_cache_file):
-                try:
-                    pathlib.Path(self._token_cache_file).unlink()
-                    logger.debug('invalidated local token cache: {}'.format(
-                        self._token_cache_file))
-                except OSError:
-                    pass
-            raise
-        return session
-
-
-def _create_aad_credentials(
-        aad_directory_id, aad_user, aad_password, endpoint, token_cache_file):
-    # type: (str, str, str, str,
-    #        str) -> azure.common.credentials.UserPassCredentials
-    """Create Azure Active Directory credentials
-    :param str aad_directory_id: aad directory/tenant id
-    :param str aad_user: aad user
-    :param str aad_password: aad password
-    :param str endpoint: management endpoint
-    :param str token_cache_file: token cache file
-    :rtype: azure.common.credentials.UserPassCredentials
-    :return: aad credentials object
-    """
-    if util.is_not_empty(aad_password):
-        try:
-            return azure.common.credentials.UserPassCredentials(
-                username=aad_user,
-                password=aad_password,
-                resource=endpoint,
-            )
-        except msrest.exceptions.AuthenticationError as e:
-            if 'AADSTS50079' in e.args[0]:
-                raise RuntimeError('{} {}'.format(
-                    e.args[0][2:],
-                    'Do not pass an AAD password to shipyard and try again.'))
-    else:
-        return DeviceCodeAuthentication(
-            context=adal.AuthenticationContext(
-                'https://login.microsoftonline.com/{}'.format(aad_directory_id)
-            ),
-            resource=endpoint,
-            client_id=_CLIENT_ID,
-            token_cache_file=token_cache_file,
-        )
-
-
-def create_clients(
-        subscription_id, aad_directory_id, aad_user, aad_password, endpoint,
-        token_cache_file):
-    # type: (str, str, str, str, str, str) ->
+def create_clients(ctx, mgmt_aad, subscription_id):
+    # type: (CliContext, settings.AADSettings, str) ->
     #        Tuple[azure.mgmt.resource.resources.ResourceManagementClient,
     #              azure.mgmt.compute.ComputeManagementClient,
     #              azure.mgmt.network.NetworkManagementClient]
     """Create resource, compute and network clients
+    :param CliContext ctx: Cli Context
+    :param settings.AADSettings mgmt_aad: AAD settings
     :param str subscription_id: subscription id
-    :param str aad_directory_id: aad directory/tenant id
-    :param str aad_user: aad user
-    :param str aad_password: aad_password
-    :param str endpoint: management endpoint
-    :param str token_cache_file: token cache file
     :rtype: tuple
     :return: (
         azure.mgmt.resource.resources.ResourceManagementClient,
         azure.mgmt.compute.ComputeManagementClient,
         azure.mgmt.network.NetworkManagementClient)
     """
-    credentials = _create_aad_credentials(
-        aad_directory_id, aad_user, aad_password, endpoint, token_cache_file)
+    if subscription_id is None:
+        return (None, None, None)
+    credentials = aad.create_aad_credentials(ctx, mgmt_aad)
     resource_client = azure.mgmt.resource.resources.ResourceManagementClient(
         credentials, subscription_id)
     compute_client = azure.mgmt.compute.ComputeManagementClient(

convoy/settings.py

@@ -77,17 +77,21 @@ SSHSettings = collections.namedtuple(
         'hpn_server_swap',
     ]
 )
+AADSettings = collections.namedtuple(
+    'AADSettings', [
+        'directory_id', 'application_id', 'auth_key', 'rsa_private_key_pem',
+        'x509_cert_sha1_thumbprint', 'user', 'password', 'endpoint',
+        'token_cache_file',
+    ]
+)
 KeyVaultCredentialsSettings = collections.namedtuple(
     'KeyVaultCredentialsSettings', [
-        'keyvault_uri', 'keyvault_credentials_secret_id', 'aad_directory_id',
-        'aad_application_id', 'aad_auth_key', 'aad_user', 'aad_password',
-        'aad_cert_private_key', 'aad_cert_thumbprint',
+        'aad', 'keyvault_uri', 'keyvault_credentials_secret_id',
     ]
 )
 ManagementCredentialsSettings = collections.namedtuple(
     'ManagementCredentialsSettings', [
-        'subscription_id', 'aad_directory_id', 'aad_user', 'aad_password',
-        'endpoint', 'token_cache_enabled', 'token_cache_file'
+        'aad', 'subscription_id',
     ]
 )
 BatchCredentialsSettings = collections.namedtuple(
@@ -582,6 +586,61 @@ def raw_credentials(config, omit_keyvault):
     return conf
 
 
+def _aad_credentials(conf, default_endpoint=None):
+    # type: (dict, str) -> AADSettings
+    """Retrieve AAD Settings
+    :param dict config: configuration object
+    :param str default_endpoint: default endpoint
+    :rtype: AADSettings
+    :return: AAD settings
+    """
+    if 'aad' in conf:
+        aad_directory_id = _kv_read_checked(conf['aad'], 'directory_id')
+        aad_application_id = _kv_read_checked(conf['aad'], 'application_id')
+        aad_auth_key = _kv_read_checked(conf['aad'], 'auth_key')
+        aad_user = _kv_read_checked(conf['aad'], 'user')
+        aad_password = _kv_read_checked(conf['aad'], 'password')
+        aad_cert_private_key = _kv_read_checked(
+            conf['aad'], 'rsa_private_key_pem')
+        aad_cert_thumbprint = _kv_read_checked(
+            conf['aad'], 'x509_cert_sha1_thumbprint')
+        aad_endpoint = _kv_read_checked(
+            conf['aad'], 'endpoint', default_endpoint)
+        if 'token_cache' not in conf['aad']:
+            conf['aad']['token_cache'] = {}
+        token_cache_enabled = _kv_read(
+            conf['aad']['token_cache'], 'enabled', True)
+        if token_cache_enabled:
+            token_cache_file = _kv_read_checked(
+                conf['aad']['token_cache'], 'filename',
+                '.batch_shipyard_aad_management_token.json')
+        else:
+            token_cache_file = None
+        return AADSettings(
+            directory_id=aad_directory_id,
+            application_id=aad_application_id,
+            auth_key=aad_auth_key,
+            user=aad_user,
+            password=aad_password,
+            rsa_private_key_pem=aad_cert_private_key,
+            x509_cert_sha1_thumbprint=aad_cert_thumbprint,
+            endpoint=aad_endpoint,
+            token_cache_file=token_cache_file,
+        )
+    else:
+        return AADSettings(
+            directory_id=None,
+            application_id=None,
+            auth_key=None,
+            user=None,
+            password=None,
+            rsa_private_key_pem=None,
+            x509_cert_sha1_thumbprint=None,
+            endpoint=default_endpoint,
+            token_cache_file=None,
+        )
+
+
 def credentials_keyvault(config):
     # type: (dict) -> KeyVaultCredentialsSettings
     """Get KeyVault settings
@@ -593,70 +652,13 @@ def credentials_keyvault(config):
         conf = config['credentials']['keyvault']
     except (KeyError, TypeError):
         conf = {}
-    try:
-        keyvault_uri = conf['uri']
-        if util.is_none_or_empty(keyvault_uri):
-            raise KeyError()
-    except KeyError:
-        keyvault_uri = None
-    try:
-        keyvault_credentials_secret_id = conf['credentials_secret_id']
-        if util.is_none_or_empty(keyvault_credentials_secret_id):
-            raise KeyError()
-    except KeyError:
-        keyvault_credentials_secret_id = None
-    try:
-        aad_directory_id = conf['aad']['directory_id']
-        if util.is_none_or_empty(aad_directory_id):
-            raise KeyError()
-    except KeyError:
-        aad_directory_id = None
-    try:
-        aad_application_id = conf['aad']['application_id']
-        if util.is_none_or_empty(aad_application_id):
-            raise KeyError()
-    except KeyError:
-        aad_application_id = None
-    try:
-        aad_auth_key = conf['aad']['auth_key']
-        if util.is_none_or_empty(aad_auth_key):
-            raise KeyError()
-    except KeyError:
-        aad_auth_key = None
-    try:
-        aad_user = conf['aad']['user']
-        if util.is_none_or_empty(aad_user):
-            raise KeyError()
-    except KeyError:
-        aad_user = None
-    try:
-        aad_password = conf['aad']['password']
-        if util.is_none_or_empty(aad_password):
-            raise KeyError()
-    except KeyError:
-        aad_password = None
-    try:
-        aad_cert_private_key = conf['aad']['rsa_private_key_pem']
-        if util.is_none_or_empty(aad_cert_private_key):
-            raise KeyError()
-    except KeyError:
-        aad_cert_private_key = None
-    try:
-        aad_cert_thumbprint = conf['aad']['x509_cert_sha1_thumbprint']
-        if util.is_none_or_empty(aad_cert_thumbprint):
-            raise KeyError()
-    except KeyError:
-        aad_cert_thumbprint = None
+    keyvault_uri = _kv_read_checked(conf, 'uri')
+    keyvault_credentials_secret_id = _kv_read_checked(
+        conf, 'credentials_secret_id')
     return KeyVaultCredentialsSettings(
+        aad=_aad_credentials(conf, default_endpoint='https://vault.azure.net'),
         keyvault_uri=keyvault_uri,
         keyvault_credentials_secret_id=keyvault_credentials_secret_id,
-        aad_directory_id=aad_directory_id,
-        aad_application_id=aad_application_id,
-        aad_auth_key=aad_auth_key,
-        aad_user=aad_user,
-        aad_password=aad_password,
-        aad_cert_private_key=aad_cert_private_key,
-        aad_cert_thumbprint=aad_cert_thumbprint,
     )
 
 
@@ -671,56 +673,11 @@ def credentials_management(config):
         conf = config['credentials']['management']
     except (KeyError, TypeError):
         conf = {}
-    try:
-        subscription_id = conf['subscription_id']
-        if util.is_none_or_empty(subscription_id):
-            raise KeyError()
-    except KeyError:
-        subscription_id = None
-    try:
-        endpoint = conf['endpoint']
-        if util.is_none_or_empty(endpoint):
-            raise KeyError()
-    except KeyError:
-        endpoint = 'https://management.core.windows.net/'
-    try:
-        token_cache_enabled = conf['token_cache']['enabled']
-    except KeyError:
-        token_cache_enabled = True
-    try:
-        token_cache_file = conf['token_cache']['filename']
-        if util.is_none_or_empty(token_cache_file):
-            raise KeyError()
-    except KeyError:
-        token_cache_file = '.batch_shipyard_aad_management_token.json'
-    if not token_cache_enabled:
-        token_cache_file = None
-    try:
-        aad_directory_id = conf['aad']['directory_id']
-        if util.is_none_or_empty(aad_directory_id):
-            raise KeyError()
-    except KeyError:
-        aad_directory_id = None
-    try:
-        aad_user = conf['aad']['user']
-        if util.is_none_or_empty(aad_user):
-            raise KeyError()
-    except KeyError:
-        aad_user = None
-    try:
-        aad_password = conf['aad']['password']
-        if util.is_none_or_empty(aad_password):
-            raise KeyError()
-    except KeyError:
-        aad_password = None
+    subscription_id = _kv_read_checked(conf, 'subscription_id')
     return ManagementCredentialsSettings(
+        aad=_aad_credentials(
+            conf, default_endpoint='https://management.core.windows.net/'),
         subscription_id=subscription_id,
-        aad_directory_id=aad_directory_id,
-        aad_user=aad_user,
-        aad_password=aad_password,
-        endpoint=endpoint,
-        token_cache_enabled=token_cache_enabled,
-        token_cache_file=token_cache_file,
     )
 
 

docs/10-batch-shipyard-configuration.md

@@ -40,7 +40,8 @@ The credentials schema is as follows:
                 "x509_cert_sha1_thumbprint": "01AB02CD...",
                 "user": "me@domain.com",
                 "password": "password"
-            }
+            },
+            "endpoint": "https://vault.azure.net"
         },
         "batch": {
             "account": "awesomebatchaccountname",
@@ -99,6 +100,11 @@ instead for AAD and KeyVault credentials.
       certificate for use with Certificate-based authentication
     * (optional) `user` AAD username
     * (optional) `password` AAD password associated with the user
+    * (optional) `endpoint` is the KeyVault AAD endpoint
+    * (optional) `token_cache` defines token cache properties for device code
+      auth
+      * (optional) `enabled` enables the token cache for device code auth
+      * (optional) `filename` specifies the file path to cache the signed token
 * (required) The `batch` property defines the Azure Batch account. Members
 under the `batch` property can be found in the
 [Azure Portal](https://portal.azure.com) under your Batch account.

docs/99-current-limitations.md

@@ -26,5 +26,3 @@ current limitation of the underlying Azure Batch service.
 This is a current limitation of the underlying VM and host drivers.
 * On-premise Docker private registries are not supported at this time due to
 VNet requirements.
-* Credential storage using Azure Active Directory and KeyVault is only
-supported for Azure Public Cloud regions.

shipyard.py

@@ -75,6 +75,7 @@ class CliContext(object):
         self.aad_password = None
         self.aad_cert_private_key = None
         self.aad_cert_thumbprint = None
+        self.aad_endpoint = None
         # management options
         self.subscription_id = None
 
@@ -141,6 +142,7 @@ class CliContext(object):
         del self.aad_password
         del self.aad_cert_private_key
         del self.aad_cert_thumbprint
+        del self.aad_endpoint
         del self.subscription_id
         self.config = None
 
@@ -456,6 +458,19 @@ def _aad_cert_thumbprint_option(f):
         callback=callback)(f)
 
 
+def _aad_endpoint_option(f):
+    def callback(ctx, param, value):
+        clictx = ctx.ensure_object(CliContext)
+        clictx.aad_endpoint = value
+        return value
+    return click.option(
+        '--aad-endpoint',
+        expose_value=False,
+        envvar='SHIPYARD_AAD_ENDPOINT',
+        help='Azure Active Directory endpoint',
+        callback=callback)(f)
+
+
 def _azure_management_subscription_id_option(f):
     def callback(ctx, param, value):
         clictx = ctx.ensure_object(CliContext)
@@ -560,13 +575,7 @@ def common_options(f):
     return f
 
 
-def batch_options(f):
-    f = _jobs_option(f)
-    f = _pool_option(f)
-    return f
-
-
-def keyvault_options(f):
+def aad_options(f):
     f = _aad_cert_thumbprint_option(f)
     f = _aad_cert_private_key_option(f)
     f = _aad_password_option(f)
@@ -574,15 +583,25 @@ def keyvault_options(f):
     f = _aad_auth_key_option(f)
     f = _aad_application_id_option(f)
     f = _aad_directory_id_option(f)
+    f = _aad_endpoint_option(f)
+    return f
+
+
+def batch_options(f):
+    f = _jobs_option(f)
+    f = _pool_option(f)
+    return f
+
+
+def keyvault_options(f):
+    f = aad_options(f)
     f = _azure_keyvault_credentials_secret_id_option(f)
     f = _azure_keyvault_uri_option(f)
     return f
 
 
 def remotefs_options(f):
-    f = _aad_password_option(f)
-    f = _aad_user_option(f)
-    f = _aad_directory_id_option(f)
+    f = aad_options(f)
     f = _azure_management_subscription_id_option(f)
     f = _remotefs_option(f)
     return f