Prep for add pool with vnet changes
- Centralize various client creation logic
This commit is contained in:
Fred Park
Родитель
8f7aee3a2f
Коммит
66d90dde90
|
|
@ -28,7 +28,7 @@
|
|||
"x509_cert_sha1_thumbprint": "",
|
||||
"user": "",
|
||||
"password": "",
|
||||
"endpoint": "https://management.core.windows.net/",
|
||||
"endpoint": "https://management.azure.com/",
|
||||
"token_cache": {
|
||||
"enabled": true,
|
||||
"filename": ""
|
||||
|
|
@ -39,6 +39,9 @@
|
|||
"account": "",
|
||||
"account_key": "",
|
||||
"account_key_keyvault_secret_id": "https://myvault.vault.azure.net/secrets/batchkey",
|
||||
"account_service_url": "",
|
||||
"user_subscription": false,
|
||||
"resource_group": "",
|
||||
"aad": {
|
||||
"directory_id": "",
|
||||
"application_id": "",
|
||||
|
|
@ -47,14 +50,12 @@
|
|||
"x509_cert_sha1_thumbprint": "",
|
||||
"user": "",
|
||||
"password": "",
|
||||
"endpoint": "https://batch.core.windows.net/",
|
||||
"endpoint": "https://batch.azure.com/",
|
||||
"token_cache": {
|
||||
"enabled": true,
|
||||
"filename": ""
|
||||
}
|
||||
},
|
||||
"account_service_url": "",
|
||||
"user_subscription": false
|
||||
}
|
||||
},
|
||||
"storage": {
|
||||
"mystorageaccount": {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,15 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"virtual_network": {
|
||||
"id": "",
|
||||
"create_nonexistant": false,
|
||||
"address_space": "",
|
||||
"subnet": {
|
||||
"id": "",
|
||||
"mask": ""
|
||||
}
|
||||
},
|
||||
"ssh": {
|
||||
"username": "docker",
|
||||
"expiry_days": 7,
|
||||
|
|
|
|||
|
|
@ -42,18 +42,14 @@ import os
|
|||
import tempfile
|
||||
import time
|
||||
# non-stdlib imports
|
||||
import azure.batch.batch_auth as batchauth
|
||||
import azure.batch.batch_service_client as batchsc
|
||||
import azure.batch.models as batchmodels
|
||||
# local imports
|
||||
from . import aad
|
||||
from . import crypto
|
||||
from . import data
|
||||
from . import keyvault
|
||||
from . import settings
|
||||
from . import storage
|
||||
from . import util
|
||||
from .version import __version__
|
||||
|
||||
# create logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -70,27 +66,6 @@ _RUN_ELEVATED = batchmodels.UserIdentity(
|
|||
)
|
||||
|
||||
|
||||
def create_client(ctx):
|
||||
# type: (CliContext) -> azure.batch.batch_service_client.BatchServiceClient
|
||||
"""Create batch client
|
||||
:param CliContext ctx: Cli Context
|
||||
:rtype: azure.batch.batch_service_client.BatchServiceClient
|
||||
:return: batch service client
|
||||
"""
|
||||
bc = settings.credentials_batch(ctx.config)
|
||||
use_aad = bc.user_subscription or util.is_none_or_empty(bc.account_key)
|
||||
if use_aad:
|
||||
batch_aad = settings.credentials_batch(ctx.config)
|
||||
credentials = aad.create_aad_credentials(ctx, batch_aad.aad)
|
||||
else:
|
||||
credentials = batchauth.SharedKeyCredentials(
|
||||
bc.account, bc.account_key)
|
||||
batch_client = batchsc.BatchServiceClient(
|
||||
credentials, base_url=bc.account_service_url)
|
||||
batch_client.config.add_user_agent('batch-shipyard/{}'.format(__version__))
|
||||
return batch_client
|
||||
|
||||
|
||||
def list_node_agent_skus(batch_client):
|
||||
# type: (batch.BatchServiceClient) -> None
|
||||
"""List all node agent skus
|
||||
|
|
|
|||
|
|
@ -0,0 +1,254 @@
|
|||
# Copyright (c) Microsoft Corporation
|
||||
#
|
||||
# All rights reserved.
|
||||
#
|
||||
# MIT License
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
# compat imports
|
||||
from __future__ import (
|
||||
absolute_import, division, print_function, unicode_literals
|
||||
)
|
||||
from builtins import ( # noqa
|
||||
bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
|
||||
next, oct, open, pow, round, super, filter, map, zip)
|
||||
# stdlib imports
|
||||
import logging
|
||||
# non-stdlib imports
|
||||
import azure.batch.batch_auth as batchauth
|
||||
import azure.batch.batch_service_client as batchsc
|
||||
import azure.keyvault
|
||||
import azure.mgmt.batch
|
||||
import azure.mgmt.compute
|
||||
import azure.mgmt.network
|
||||
import azure.mgmt.resource
|
||||
import azure.storage.blob as azureblob
|
||||
import azure.storage.queue as azurequeue
|
||||
import azure.storage.table as azuretable
|
||||
# local imports
|
||||
from . import aad
|
||||
from . import settings
|
||||
from . import storage
|
||||
from . import util
|
||||
from .version import __version__
|
||||
|
||||
# create logger
|
||||
logger = logging.getLogger(__name__)
|
||||
util.setup_logger(logger)
|
||||
|
||||
|
||||
def create_resource_client(ctx, credentials=None, subscription_id=None):
|
||||
# type: (CliContext, object, str) ->
|
||||
# azure.mgmt.resource.resources.ResourceManagementClient
|
||||
"""Create resource management client
|
||||
:param CliContext ctx: Cli Context
|
||||
:param object credentials: credentials object
|
||||
:param str subscription_id: subscription id
|
||||
:rtype: azure.mgmt.resource.resources.ResourceManagementClient
|
||||
:return: resource management client
|
||||
"""
|
||||
mgmt_aad = None
|
||||
if credentials is None:
|
||||
mgmt_aad = settings.credentials_management(ctx.config).aad
|
||||
credentials = aad.create_aad_credentials(ctx, mgmt_aad)
|
||||
if util.is_none_or_empty(subscription_id):
|
||||
if mgmt_aad is None:
|
||||
mgmt_aad = settings.credentials_management(ctx.config).aad
|
||||
subscription_id = ctx.subscription_id or mgmt_aad.subscription_id
|
||||
return azure.mgmt.resource.resources.ResourceManagementClient(
|
||||
credentials, subscription_id)
|
||||
|
||||
|
||||
def create_compute_client(ctx, credentials=None, subscription_id=None):
|
||||
# type: (CliContext, object, str) ->
|
||||
# azure.mgmt.compute.ComputeManagementClient
|
||||
"""Create compute management client
|
||||
:param CliContext ctx: Cli Context
|
||||
:param object credentials: credentials object
|
||||
:param str subscription_id: subscription id
|
||||
:rtype: azure.mgmt.compute.ComputeManagementClient
|
||||
:return: compute management client
|
||||
"""
|
||||
mgmt_aad = None
|
||||
if credentials is None:
|
||||
mgmt_aad = settings.credentials_management(ctx.config).aad
|
||||
credentials = aad.create_aad_credentials(ctx, mgmt_aad)
|
||||
if util.is_none_or_empty(subscription_id):
|
||||
if mgmt_aad is None:
|
||||
mgmt_aad = settings.credentials_management(ctx.config).aad
|
||||
subscription_id = ctx.subscription_id or mgmt_aad.subscription_id
|
||||
return azure.mgmt.compute.ComputeManagementClient(
|
||||
credentials, subscription_id)
|
||||
|
||||
|
||||
def create_network_client(ctx, credentials=None, subscription_id=None):
|
||||
# type: (CliContext, object, str) ->
|
||||
# azure.mgmt.network.NetworkManagementClient
|
||||
"""Create network management client
|
||||
:param CliContext ctx: Cli Context
|
||||
:rtype: azure.mgmt.network.NetworkManagementClient
|
||||
:return: network management client
|
||||
"""
|
||||
mgmt_aad = None
|
||||
if credentials is None:
|
||||
mgmt_aad = settings.credentials_management(ctx.config).aad
|
||||
credentials = aad.create_aad_credentials(ctx, mgmt_aad)
|
||||
if util.is_none_or_empty(subscription_id):
|
||||
if mgmt_aad is None:
|
||||
mgmt_aad = settings.credentials_management(ctx.config).aad
|
||||
subscription_id = ctx.subscription_id or mgmt_aad.subscription_id
|
||||
return azure.mgmt.network.NetworkManagementClient(
|
||||
credentials, subscription_id)
|
||||
|
||||
|
||||
def create_arm_clients(ctx, batch_clients=False):
|
||||
# type: (CliContext, bool) ->
|
||||
# Tuple[azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
# azure.mgmt.compute.ComputeManagementClient,
|
||||
# azure.mgmt.network.NetworkManagementClient,
|
||||
# azure.mgmt.batch.BatchManagementClient,
|
||||
# azure.batch.batch_service_client.BatchServiceClient]
|
||||
"""Create resource, compute and network clients
|
||||
:param CliContext ctx: Cli Context
|
||||
:param bool batch_clients: create batch clients
|
||||
:rtype: tuple
|
||||
:return: (
|
||||
azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
azure.mgmt.compute.ComputeManagementClient,
|
||||
azure.mgmt.network.NetworkManagementClient,
|
||||
azure.mgmt.batch.BatchManagementClient,
|
||||
azure.batch.batch_service_client.BatchServiceClient)
|
||||
"""
|
||||
mgmt_aad = settings.credentials_management(ctx.config).aad
|
||||
subscription_id = ctx.subscription_id or mgmt_aad.subscription_id
|
||||
if util.is_none_or_empty(subscription_id):
|
||||
return (None, None, None, None, None)
|
||||
credentials = aad.create_aad_credentials(ctx, mgmt_aad)
|
||||
resource_client = create_resource_client(
|
||||
ctx, credentials=credentials, subscription_id=subscription_id)
|
||||
compute_client = create_compute_client(
|
||||
ctx, credentials=credentials, subscription_id=subscription_id)
|
||||
network_client = create_network_client(
|
||||
ctx, credentials=credentials, subscription_id=subscription_id)
|
||||
if batch_clients:
|
||||
batch_mgmt_client, batch_client = create_batch_clients(ctx)
|
||||
else:
|
||||
batch_mgmt_client = None
|
||||
batch_client = None
|
||||
return (
|
||||
resource_client, compute_client, network_client, batch_mgmt_client,
|
||||
batch_client
|
||||
)
|
||||
|
||||
|
||||
def create_keyvault_client(ctx):
|
||||
# type: (CliContext) -> azure.keyvault.KeyVaultClient
|
||||
"""Create KeyVault client
|
||||
:param CliContext ctx: Cli Context
|
||||
:rtype: azure.keyvault.KeyVaultClient
|
||||
:return: keyvault client
|
||||
"""
|
||||
kv_aad = settings.credentials_keyvault(ctx.config).aad
|
||||
return azure.keyvault.KeyVaultClient(
|
||||
aad.create_aad_credentials(ctx, kv_aad)
|
||||
)
|
||||
|
||||
|
||||
def create_batch_mgmt_client(ctx, credentials=None, subscription_id=None):
|
||||
# type: (CliContext, object, str) ->
|
||||
# azure.mgmt.batch.BatchManagementClient
|
||||
"""Create batch management client
|
||||
:param CliContext ctx: Cli Context
|
||||
:param object credentials: credentials object
|
||||
:param str subscription_id: subscription id
|
||||
:rtype: azure.mgmt.batch.BatchManagementClient
|
||||
:return: batch management client
|
||||
"""
|
||||
batch_aad = None
|
||||
if credentials is None:
|
||||
batch_aad = settings.credentials_batch(ctx.config).aad
|
||||
credentials = aad.create_aad_credentials(ctx, batch_aad)
|
||||
if util.is_none_or_empty(subscription_id):
|
||||
if batch_aad is None:
|
||||
batch_aad = settings.credentials_batch(ctx.config).aad
|
||||
subscription_id = ctx.subscription_id or batch_aad.subscription_id
|
||||
if util.is_none_or_empty(subscription_id):
|
||||
return None
|
||||
batch_mgmt_client = azure.mgmt.batch.BatchManagementClient(
|
||||
credentials, subscription_id)
|
||||
batch_mgmt_client.config.add_user_agent(
|
||||
'batch-shipyard/{}'.format(__version__))
|
||||
return batch_mgmt_client
|
||||
|
||||
|
||||
def create_batch_clients(ctx):
|
||||
# type: (CliContext) ->
|
||||
# Tuple[azure.mgmt.batch.BatchManagementClient,
|
||||
# azure.batch.batch_service_client.BatchServiceClient]
|
||||
"""Create batch client
|
||||
:param CliContext ctx: Cli Context
|
||||
:rtype: tuple
|
||||
:return: (
|
||||
azure.mgmt.batch.BatchManagementClient,
|
||||
azure.batch.batch_service_client.BatchServiceClient)
|
||||
"""
|
||||
bc = settings.credentials_batch(ctx.config)
|
||||
use_aad = bc.user_subscription or util.is_none_or_empty(bc.account_key)
|
||||
batch_mgmt_client = None
|
||||
if use_aad:
|
||||
subscription_id = ctx.subscription_id or bc.subscription_id
|
||||
batch_aad = settings.credentials_batch(ctx.config).aad
|
||||
credentials = aad.create_aad_credentials(ctx, batch_aad)
|
||||
batch_mgmt_client = create_batch_mgmt_client(
|
||||
ctx, credentials=credentials, subscription_id=subscription_id)
|
||||
else:
|
||||
credentials = batchauth.SharedKeyCredentials(
|
||||
bc.account, bc.account_key)
|
||||
batch_client = batchsc.BatchServiceClient(
|
||||
credentials, base_url=bc.account_service_url)
|
||||
batch_client.config.add_user_agent('batch-shipyard/{}'.format(__version__))
|
||||
return (batch_mgmt_client, batch_client)
|
||||
|
||||
|
||||
def create_storage_clients():
|
||||
# type: (None) -> tuple
|
||||
"""Create storage clients
|
||||
:rtype: tuple
|
||||
:return: blob_client, queue_client, table_client
|
||||
"""
|
||||
account_name = storage.get_storageaccount()
|
||||
account_key = storage.get_storageaccount_key()
|
||||
endpoint_suffix = storage.get_storageaccount_endpoint()
|
||||
blob_client = azureblob.BlockBlobService(
|
||||
account_name=account_name,
|
||||
account_key=account_key,
|
||||
endpoint_suffix=endpoint_suffix,
|
||||
)
|
||||
queue_client = azurequeue.QueueService(
|
||||
account_name=account_name,
|
||||
account_key=account_key,
|
||||
endpoint_suffix=endpoint_suffix,
|
||||
)
|
||||
table_client = azuretable.TableService(
|
||||
account_name=account_name,
|
||||
account_key=account_key,
|
||||
endpoint_suffix=endpoint_suffix,
|
||||
)
|
||||
return blob_client, queue_client, table_client
|
||||
214
convoy/fleet.py
214
convoy/fleet.py
|
|
@ -142,7 +142,7 @@ _PERF_FILE = (
|
|||
)
|
||||
|
||||
|
||||
def _adjust_general_settings(config):
|
||||
def adjust_general_settings(config):
|
||||
# type: (dict) -> None
|
||||
"""Adjust general settings
|
||||
:param dict config: configuration dict
|
||||
|
|
@ -187,7 +187,7 @@ def _adjust_general_settings(config):
|
|||
settings.set_batch_shipyard_encryption_enabled(config, False)
|
||||
|
||||
|
||||
def _populate_global_settings(config, fs_context):
|
||||
def populate_global_settings(config, fs_context):
|
||||
# type: (dict, bool) -> None
|
||||
"""Populate global settings from config
|
||||
:param dict config: configuration dict
|
||||
|
|
@ -211,65 +211,6 @@ def _populate_global_settings(config, fs_context):
|
|||
bs.generated_sas_expiry_days)
|
||||
|
||||
|
||||
def _create_clients(ctx, fs_context):
|
||||
# type: (CliContext, bool) -> tuple
|
||||
"""Create authenticated clients
|
||||
:param CliContext ctx: Cli Context
|
||||
:param bool fs_context: only initialize storage clients
|
||||
:rtype: tuple
|
||||
:return: (batch client, blob client, queue client, table client)
|
||||
"""
|
||||
if fs_context:
|
||||
batch_client = None
|
||||
else:
|
||||
batch_client = batch.create_client(ctx)
|
||||
blob_client, queue_client, table_client = storage.create_clients()
|
||||
return batch_client, blob_client, queue_client, table_client
|
||||
|
||||
|
||||
def create_keyvault_client(ctx):
|
||||
# type: (CliContext) -> azure.keyvault.KeyVaultClient
|
||||
"""Create KeyVault client
|
||||
:param CliContext ctx: Cli Context
|
||||
:rtype: azure.keyvault.KeyVaultClient
|
||||
:return: key vault client
|
||||
"""
|
||||
kv = settings.credentials_keyvault(ctx.config)
|
||||
return keyvault.create_client(ctx, kv.aad)
|
||||
|
||||
|
||||
def create_fs_clients(ctx):
|
||||
# type: (CliContext) ->
|
||||
# Tuple[azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
# azure.mgmt.compute.ComputeManagementClient,
|
||||
# azure.mgmt.network.NetworkManagementClient]
|
||||
"""Create clients needed for fs: resource management, compute, network
|
||||
:param CliContext ctx: Cli Context
|
||||
:rtype: tuple
|
||||
:return: (
|
||||
azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
azure.mgmt.compute.ComputeManagementClient,
|
||||
azure.mgmt.network.NetworkManagementClient)
|
||||
"""
|
||||
mgmt = settings.credentials_management(ctx.config)
|
||||
subscription_id = ctx.subscription_id or mgmt.subscription_id
|
||||
return remotefs.create_clients(ctx, mgmt.aad, subscription_id)
|
||||
|
||||
|
||||
def initialize(ctx, fs_context=False):
|
||||
# type: (CliContext, bool) -> tuple
|
||||
"""Initialize fleet and create authenticated clients
|
||||
:param CliContext ctx: Cli Context
|
||||
:param bool fs_context: only initialize storage clients
|
||||
:rtype: tuple
|
||||
:return: (batch client, blob client, queue client, table client)
|
||||
"""
|
||||
if not fs_context:
|
||||
_adjust_general_settings(ctx.config)
|
||||
_populate_global_settings(ctx.config, fs_context)
|
||||
return _create_clients(ctx, fs_context)
|
||||
|
||||
|
||||
def fetch_credentials_json_from_keyvault(
|
||||
keyvault_client, keyvault_uri, keyvault_credentials_secret_id):
|
||||
# type: (azure.keyvault.KeyVaultClient, str, str) -> dict
|
||||
|
|
@ -453,14 +394,42 @@ def _setup_azurefile_volume_driver(blob_client, config):
|
|||
return bin, srv, srvenv, volcreate
|
||||
|
||||
|
||||
def _add_pool(batch_client, blob_client, config):
|
||||
# type: (batchsc.BatchServiceClient, azureblob.BlockBlobService,
|
||||
# dict) -> None
|
||||
def _add_pool(
|
||||
resource_client, network_client, batch_client, blob_client, config):
|
||||
# type: (azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
# azure.mgmt.network.NetworkManagementClient,
|
||||
# azure.mgmt.batch.BatchManagementClient,
|
||||
# azure.batch.batch_service_client.BatchServiceClient,
|
||||
# azureblob.BlockBlobService, dict) -> None
|
||||
"""Add a Batch pool to account
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.mgmt.resource.resources.ResourceManagementClient
|
||||
resource_client: resource client
|
||||
:param azure.mgmt.network.NetworkManagementClient network_client:
|
||||
network client
|
||||
:param azure.mgmt.batch.BatchManagementClient: batch_mgmt_client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
# check shared data volume mounts before proceeding to allocate
|
||||
azurefile_vd = False
|
||||
gluster = False
|
||||
try:
|
||||
sdv = settings.global_resources_shared_data_volumes(config)
|
||||
for sdvkey in sdv:
|
||||
if settings.is_shared_data_volume_azure_file(sdv, sdvkey):
|
||||
azurefile_vd = True
|
||||
elif settings.is_shared_data_volume_gluster(sdv, sdvkey):
|
||||
gluster = True
|
||||
else:
|
||||
raise ValueError('Unknown shared data volume: {}'.format(
|
||||
settings.shared_data_volume_driver(sdv, sdvkey)))
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# TODO check virtual network settings
|
||||
|
||||
# add encryption cert to account if specified
|
||||
encrypt = settings.batch_shipyard_encryption_enabled(config)
|
||||
if encrypt:
|
||||
|
|
@ -490,21 +459,6 @@ def _add_pool(batch_client, blob_client, config):
|
|||
dr.peer_to_peer.direct_download_seed_bias,
|
||||
dr.peer_to_peer.compression,
|
||||
preg.allow_public_docker_hub_pull_on_missing)
|
||||
# check shared data volume mounts
|
||||
azurefile_vd = False
|
||||
gluster = False
|
||||
try:
|
||||
sdv = settings.global_resources_shared_data_volumes(config)
|
||||
for sdvkey in sdv:
|
||||
if settings.is_shared_data_volume_azure_file(sdv, sdvkey):
|
||||
azurefile_vd = True
|
||||
elif settings.is_shared_data_volume_gluster(sdv, sdvkey):
|
||||
gluster = True
|
||||
else:
|
||||
raise ValueError('Unknown shared data volume: {}'.format(
|
||||
settings.shared_data_volume_driver(sdv, sdvkey)))
|
||||
except KeyError:
|
||||
pass
|
||||
# create resource files list
|
||||
_rflist = [_NODEPREP_FILE, _JOBPREP_FILE, _BLOBXFER_FILE]
|
||||
if not bs.use_shipyard_docker_image:
|
||||
|
|
@ -1360,17 +1314,29 @@ def action_cert_del(batch_client, config):
|
|||
def action_pool_listskus(batch_client):
|
||||
# type: (batchsc.BatchServiceClient) -> None
|
||||
"""Action: Pool Listskus
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
"""
|
||||
batch.list_node_agent_skus(batch_client)
|
||||
|
||||
|
||||
def action_pool_add(
|
||||
batch_client, blob_client, queue_client, table_client, config):
|
||||
# type: (batchsc.BatchServiceClient, azureblob.BlockBlobService,
|
||||
# azurequeue.QueueService, azuretable.TableService, dict) -> None
|
||||
resource_client, network_client, batch_mgmt_client, batch_client,
|
||||
blob_client, queue_client, table_client, config):
|
||||
# type: (azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
# azure.mgmt.network.NetworkManagementClient,
|
||||
# azure.mgmt.batch.BatchManagementClient,
|
||||
# azure.batch.batch_service_client.BatchServiceClient,
|
||||
# azureblob.BlockBlobService, azurequeue.QueueService,
|
||||
# azuretable.TableService, dict) -> None
|
||||
"""Action: Pool Add
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.mgmt.resource.resources.ResourceManagementClient
|
||||
resource_client: resource client
|
||||
:param azure.mgmt.network.NetworkManagementClient network_client:
|
||||
network client
|
||||
:param azure.mgmt.batch.BatchManagementClient: batch_mgmt_client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.queue.QueueService queue_client: queue client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
|
|
@ -1387,13 +1353,17 @@ def action_pool_add(
|
|||
blob_client, queue_client, table_client, config)
|
||||
_adjust_settings_for_pool_creation(config)
|
||||
storage.populate_queues(queue_client, table_client, config)
|
||||
_add_pool(batch_client, blob_client, config)
|
||||
_add_pool(
|
||||
resource_client, network_client, batch_mgmt_client, batch_client,
|
||||
blob_client, config
|
||||
)
|
||||
|
||||
|
||||
def action_pool_list(batch_client):
|
||||
# type: (batchsc.BatchServiceClient) -> None
|
||||
"""Action: Pool List
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
"""
|
||||
batch.list_pools(batch_client)
|
||||
|
||||
|
|
@ -1405,7 +1375,8 @@ def action_pool_delete(
|
|||
# azurequeue.QueueService, azuretable.TableService, dict,
|
||||
# bool) -> None
|
||||
"""Action: Pool Delete
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.queue.QueueService queue_client: queue client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
|
|
@ -1433,8 +1404,8 @@ def action_pool_resize(batch_client, blob_client, config, wait):
|
|||
# type: (batchsc.BatchServiceClient, azureblob.BlockBlobService,
|
||||
# dict, bool) -> None
|
||||
"""Resize pool that may contain glusterfs
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param dict config: configuration dict
|
||||
:param bool wait: wait for operation to complete
|
||||
|
|
@ -1543,7 +1514,8 @@ def action_pool_resize(batch_client, blob_client, config, wait):
|
|||
def action_pool_grls(batch_client, config):
|
||||
# type: (batchsc.BatchServiceClient, dict) -> None
|
||||
"""Action: Pool Grls
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
batch.get_remote_login_settings(batch_client, config)
|
||||
|
|
@ -1554,7 +1526,8 @@ def action_pool_grls(batch_client, config):
|
|||
def action_pool_listnodes(batch_client, config):
|
||||
# type: (batchsc.BatchServiceClient, dict) -> None
|
||||
"""Action: Pool Listnodes
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
batch.list_nodes(batch_client, config)
|
||||
|
|
@ -1563,7 +1536,8 @@ def action_pool_listnodes(batch_client, config):
|
|||
def action_pool_asu(batch_client, config):
|
||||
# type: (batchsc.BatchServiceClient, dict) -> None
|
||||
"""Action: Pool Asu
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
batch.add_ssh_user(batch_client, config)
|
||||
|
|
@ -1573,7 +1547,8 @@ def action_pool_asu(batch_client, config):
|
|||
def action_pool_dsu(batch_client, config):
|
||||
# type: (batchsc.BatchServiceClient, dict) -> None
|
||||
"""Action: Pool Dsu
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
batch.del_ssh_user(batch_client, config)
|
||||
|
|
@ -1582,7 +1557,8 @@ def action_pool_dsu(batch_client, config):
|
|||
def action_pool_ssh(batch_client, config, cardinal, nodeid):
|
||||
# type: (batchsc.BatchServiceClient, dict, int, str) -> None
|
||||
"""Action: Pool Ssh
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param int cardinal: cardinal node num
|
||||
:param str nodeid: node id
|
||||
|
|
@ -1612,7 +1588,8 @@ def action_pool_ssh(batch_client, config, cardinal, nodeid):
|
|||
def action_pool_delnode(batch_client, config, nodeid):
|
||||
# type: (batchsc.BatchServiceClient, dict, str) -> None
|
||||
"""Action: Pool Delnode
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param str nodeid: nodeid to delete
|
||||
"""
|
||||
|
|
@ -1623,7 +1600,8 @@ def action_pool_rebootnode(
|
|||
batch_client, config, all_start_task_failed, nodeid):
|
||||
# type: (batchsc.BatchServiceClient, dict, bool, str) -> None
|
||||
"""Action: Pool Rebootnode
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param bool all_start_task_failed: reboot all start task failed nodes
|
||||
:param str nodeid: nodeid to reboot
|
||||
|
|
@ -1638,7 +1616,8 @@ def action_pool_rebootnode(
|
|||
def action_pool_udi(batch_client, config, image, digest):
|
||||
# type: (batchsc.BatchServiceClient, dict, str, str) -> None
|
||||
"""Action: Pool Udi
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param str image: image to update
|
||||
:param str digest: digest to update to
|
||||
|
|
@ -1654,7 +1633,8 @@ def action_jobs_add(
|
|||
# type: (batchsc.BatchServiceClient, azureblob.BlockBlobService,
|
||||
# azure.keyvault.KeyVaultClient, dict, bool, str) -> None
|
||||
"""Action: Jobs Add
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.keyvault.KeyVaultClient keyvault_client: keyvault client
|
||||
:param dict config: configuration dict
|
||||
|
|
@ -1669,7 +1649,8 @@ def action_jobs_add(
|
|||
def action_jobs_list(batch_client, config):
|
||||
# type: (batchsc.BatchServiceClient, dict) -> None
|
||||
"""Action: Jobs List
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
batch.list_jobs(batch_client, config)
|
||||
|
|
@ -1678,7 +1659,8 @@ def action_jobs_list(batch_client, config):
|
|||
def action_jobs_listtasks(batch_client, config, jobid):
|
||||
# type: (batchsc.BatchServiceClient, dict, str) -> None
|
||||
"""Action: Jobs Listtasks
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
batch.list_tasks(batch_client, config, jobid)
|
||||
|
|
@ -1687,7 +1669,8 @@ def action_jobs_listtasks(batch_client, config, jobid):
|
|||
def action_jobs_termtasks(batch_client, config, jobid, taskid, wait, force):
|
||||
# type: (batchsc.BatchServiceClient, dict, str, str, bool, bool) -> None
|
||||
"""Action: Jobs Termtasks
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param str jobid: job id
|
||||
:param str taskid: task id
|
||||
|
|
@ -1708,7 +1691,8 @@ def action_jobs_termtasks(batch_client, config, jobid, taskid, wait, force):
|
|||
def action_jobs_deltasks(batch_client, config, jobid, taskid, wait):
|
||||
# type: (batchsc.BatchServiceClient, dict, str, str, bool) -> None
|
||||
"""Action: Jobs Deltasks
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param str jobid: job id
|
||||
:param str taskid: task id
|
||||
|
|
@ -1725,7 +1709,8 @@ def action_jobs_deltasks(batch_client, config, jobid, taskid, wait):
|
|||
def action_jobs_term(batch_client, config, all, jobid, termtasks, wait):
|
||||
# type: (batchsc.BatchServiceClient, dict, bool, str, bool, bool) -> None
|
||||
"""Action: Jobs Term
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param bool all: all jobs
|
||||
:param str jobid: job id
|
||||
|
|
@ -1745,7 +1730,8 @@ def action_jobs_term(batch_client, config, all, jobid, termtasks, wait):
|
|||
def action_jobs_del(batch_client, config, all, jobid, termtasks, wait):
|
||||
# type: (batchsc.BatchServiceClient, dict, bool, str, bool, bool) -> None
|
||||
"""Action: Jobs Del
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param bool all: all jobs
|
||||
:param str jobid: job id
|
||||
|
|
@ -1765,7 +1751,8 @@ def action_jobs_del(batch_client, config, all, jobid, termtasks, wait):
|
|||
def action_jobs_cmi(batch_client, config, delete):
|
||||
# type: (batchsc.BatchServiceClient, dict, bool) -> None
|
||||
"""Action: Jobs Cmi
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param bool delete: delete all cmi jobs
|
||||
"""
|
||||
|
|
@ -1805,7 +1792,8 @@ def action_storage_clear(blob_client, queue_client, table_client, config):
|
|||
def action_data_stream(batch_client, config, filespec, disk):
|
||||
# type: (batchsc.BatchServiceClient, dict, str, bool) -> None
|
||||
"""Action: Data Stream
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param str filespec: filespec of file to retrieve
|
||||
:param bool disk: write streamed data to disk instead
|
||||
|
|
@ -1816,7 +1804,8 @@ def action_data_stream(batch_client, config, filespec, disk):
|
|||
def action_data_listfiles(batch_client, config, jobid, taskid):
|
||||
# type: (batchsc.BatchServiceClient, dict, str, str) -> None
|
||||
"""Action: Data Listfiles
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param str jobid: job id to list
|
||||
:param str taskid: task id to list
|
||||
|
|
@ -1831,7 +1820,8 @@ def action_data_listfiles(batch_client, config, jobid, taskid):
|
|||
def action_data_getfile(batch_client, config, all, filespec):
|
||||
# type: (batchsc.BatchServiceClient, dict, bool, str) -> None
|
||||
"""Action: Data Getfile
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param bool all: retrieve all files
|
||||
:param str filespec: filespec of file to retrieve
|
||||
|
|
@ -1845,7 +1835,8 @@ def action_data_getfile(batch_client, config, all, filespec):
|
|||
def action_data_getfilenode(batch_client, config, all, nodeid):
|
||||
# type: (batchsc.BatchServiceClient, dict, bool, str) -> None
|
||||
"""Action: Data Getfilenode
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param bool all: retrieve all files
|
||||
:param str nodeid: node id to retrieve file from
|
||||
|
|
@ -1859,7 +1850,8 @@ def action_data_getfilenode(batch_client, config, all, nodeid):
|
|||
def action_data_ingress(batch_client, config):
|
||||
# type: (batchsc.BatchServiceClient, dict) -> None
|
||||
"""Action: Data Ingress
|
||||
:param azure.batch.batch_service_client.BatchServiceClient: batch client
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
pool_cd = None
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ import zlib
|
|||
import azure.common.credentials
|
||||
import azure.keyvault
|
||||
# local imports
|
||||
from . import aad
|
||||
from . import settings
|
||||
from . import util
|
||||
|
||||
|
|
@ -49,20 +48,6 @@ _SECRET_ENCODED_FORMAT_KEY = 'format'
|
|||
_SECRET_ENCODED_FORMAT_VALUE = 'zlib+base64'
|
||||
|
||||
|
||||
def create_client(ctx, kv_aad):
|
||||
# type: (CliContext, settings.AADSettings) ->
|
||||
# azure.keyvault.KeyVaultClient
|
||||
"""Create KeyVault client
|
||||
:param CliContext ctx: Cli Context
|
||||
:param settings.AADSettings kv_aad: AAD settings
|
||||
:rtype: azure.keyvault.KeyVaultClient
|
||||
:return: keyvault client
|
||||
"""
|
||||
return azure.keyvault.KeyVaultClient(
|
||||
aad.create_aad_credentials(ctx, kv_aad)
|
||||
)
|
||||
|
||||
|
||||
def fetch_credentials_json(
|
||||
client, keyvault_uri, keyvault_credentials_secret_id):
|
||||
# type: (azure.keyvault.KeyVaultClient, str, str) -> dict
|
||||
|
|
|
|||
|
|
@ -38,16 +38,11 @@ try:
|
|||
except ImportError:
|
||||
import pathlib
|
||||
# non-stdlib imports
|
||||
import azure.common.credentials
|
||||
import azure.mgmt.compute
|
||||
import azure.mgmt.compute.models as computemodels
|
||||
import azure.mgmt.network
|
||||
import azure.mgmt.network.models as networkmodels
|
||||
import azure.mgmt.resource
|
||||
import azure.mgmt.resource.resources.models as rgmodels
|
||||
import msrestazure.azure_exceptions
|
||||
# local imports
|
||||
from . import aad
|
||||
from . import crypto
|
||||
from . import settings
|
||||
from . import storage
|
||||
|
|
@ -60,33 +55,6 @@ util.setup_logger(logger)
|
|||
_SSH_KEY_PREFIX = 'id_rsa_shipyard_remotefs'
|
||||
|
||||
|
||||
def create_clients(ctx, mgmt_aad, subscription_id):
|
||||
# type: (CliContext, settings.AADSettings, str) ->
|
||||
# Tuple[azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
# azure.mgmt.compute.ComputeManagementClient,
|
||||
# azure.mgmt.network.NetworkManagementClient]
|
||||
"""Create resource, compute and network clients
|
||||
:param CliContext ctx: Cli Context
|
||||
:param settings.AADSettings mgmt_aad: AAD settings
|
||||
:param str subscription_id: subscription id
|
||||
:rtype: tuple
|
||||
:return: (
|
||||
azure.mgmt.resource.resources.ResourceManagementClient,
|
||||
azure.mgmt.compute.ComputeManagementClient,
|
||||
azure.mgmt.network.NetworkManagementClient)
|
||||
"""
|
||||
if subscription_id is None:
|
||||
return (None, None, None)
|
||||
credentials = aad.create_aad_credentials(ctx, mgmt_aad)
|
||||
resource_client = azure.mgmt.resource.resources.ResourceManagementClient(
|
||||
credentials, subscription_id)
|
||||
compute_client = azure.mgmt.compute.ComputeManagementClient(
|
||||
credentials, subscription_id)
|
||||
network_client = azure.mgmt.network.NetworkManagementClient(
|
||||
credentials, subscription_id)
|
||||
return (resource_client, compute_client, network_client)
|
||||
|
||||
|
||||
def _create_managed_disk_async(compute_client, rfs, disk_name):
|
||||
# type: (azure.mgmt.compute.ComputeManagementClient,
|
||||
# settings.RemoteFsSettings, str) ->
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ ManagementCredentialsSettings = collections.namedtuple(
|
|||
BatchCredentialsSettings = collections.namedtuple(
|
||||
'BatchCredentialsSettings', [
|
||||
'aad', 'account', 'account_key', 'account_service_url',
|
||||
'user_subscription',
|
||||
'user_subscription', 'resource_group', 'subscription_id', 'location',
|
||||
]
|
||||
)
|
||||
StorageCredentialsSettings = collections.namedtuple(
|
||||
|
|
@ -176,6 +176,7 @@ ManagedDisksSettings = collections.namedtuple(
|
|||
VirtualNetworkSettings = collections.namedtuple(
|
||||
'VirtualNetworkSettings', [
|
||||
'id', 'address_space', 'subnet_id', 'subnet_mask', 'existing_ok',
|
||||
'create_nonexistant',
|
||||
]
|
||||
)
|
||||
FileServerSettings = collections.namedtuple(
|
||||
|
|
@ -696,7 +697,17 @@ def credentials_batch(config):
|
|||
"""
|
||||
conf = config['credentials']['batch']
|
||||
account_key = _kv_read_checked(conf, 'account_key')
|
||||
account_service_url = conf['account_service_url']
|
||||
user_subscription = _kv_read(conf, 'user_subscription', False)
|
||||
resource_group = _kv_read_checked(conf, 'resource_group')
|
||||
# get subscription id from management section
|
||||
try:
|
||||
subscription_id = _kv_read_checked(
|
||||
config['credentials']['management'], 'subscription_id')
|
||||
except (KeyError, TypeError):
|
||||
subscription_id = None
|
||||
# parse location from url
|
||||
location = account_service_url.split('.')[1]
|
||||
return BatchCredentialsSettings(
|
||||
aad=_aad_credentials(
|
||||
conf, default_endpoint='https://batch.core.windows.net/'),
|
||||
|
|
@ -704,6 +715,9 @@ def credentials_batch(config):
|
|||
account_key=account_key,
|
||||
account_service_url=conf['account_service_url'],
|
||||
user_subscription=user_subscription,
|
||||
resource_group=resource_group,
|
||||
location=location,
|
||||
subscription_id=subscription_id,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -2340,6 +2354,7 @@ def remotefs_settings(config):
|
|||
subnet_id=sc_vnet_subnet_id,
|
||||
subnet_mask=sc_vnet_subnet_mask,
|
||||
existing_ok=sc_vnet_existing_ok,
|
||||
create_nonexistant=True,
|
||||
),
|
||||
network_security=NetworkSecuritySettings(
|
||||
inbound=sc_ns_inbound,
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@ except ImportError:
|
|||
import azure.common
|
||||
import azure.storage.blob as azureblob
|
||||
import azure.storage.file as azurefile
|
||||
import azure.storage.queue as azurequeue
|
||||
import azure.storage.table as azuretable
|
||||
# local imports
|
||||
from . import settings
|
||||
|
|
@ -141,27 +140,6 @@ def get_storageaccount_endpoint():
|
|||
return _STORAGEACCOUNTEP
|
||||
|
||||
|
||||
def create_clients():
|
||||
# type: (None) -> tuple
|
||||
"""Create storage clients
|
||||
:rtype: tuple
|
||||
:return: blob_client, queue_client, table_client
|
||||
"""
|
||||
blob_client = azureblob.BlockBlobService(
|
||||
account_name=_STORAGEACCOUNT,
|
||||
account_key=_STORAGEACCOUNTKEY,
|
||||
endpoint_suffix=_STORAGEACCOUNTEP)
|
||||
queue_client = azurequeue.QueueService(
|
||||
account_name=_STORAGEACCOUNT,
|
||||
account_key=_STORAGEACCOUNTKEY,
|
||||
endpoint_suffix=_STORAGEACCOUNTEP)
|
||||
table_client = azuretable.TableService(
|
||||
account_name=_STORAGEACCOUNT,
|
||||
account_key=_STORAGEACCOUNTKEY,
|
||||
endpoint_suffix=_STORAGEACCOUNTEP)
|
||||
return blob_client, queue_client, table_client
|
||||
|
||||
|
||||
def create_blob_container_saskey(
|
||||
storage_settings, container, kind, create_container=False):
|
||||
# type: (StorageCredentialsSettings, str, str, bool) -> str
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
adal==0.4.5
|
||||
azure-batch==2.0.0
|
||||
azure-keyvault==0.1.0
|
||||
azure-mgmt-batch==2.0.0
|
||||
azure-mgmt-compute==0.33.0
|
||||
azure-mgmt-network==0.30.0
|
||||
azure-mgmt-resource==0.31.0
|
||||
|
|
|
|||
66
shipyard.py
66
shipyard.py
|
|
@ -39,6 +39,7 @@ except ImportError:
|
|||
# non-stdlib imports
|
||||
import click
|
||||
# local imports
|
||||
import convoy.clients
|
||||
import convoy.fleet
|
||||
import convoy.settings
|
||||
import convoy.util
|
||||
|
|
@ -57,6 +58,7 @@ class CliContext(object):
|
|||
self.verbose = False
|
||||
self.yes = False
|
||||
self.config = None
|
||||
self.batch_mgmt_client = None
|
||||
self.batch_client = None
|
||||
self.blob_client = None
|
||||
self.queue_client = None
|
||||
|
|
@ -86,16 +88,14 @@ class CliContext(object):
|
|||
"""
|
||||
self._read_credentials_config()
|
||||
self._set_global_cli_options()
|
||||
self.resource_client, self.compute_client, self.network_client = \
|
||||
convoy.fleet.create_fs_clients(self)
|
||||
self._init_config(
|
||||
skip_global_config=False, skip_pool_config=True,
|
||||
skip_fs_config=False)
|
||||
clients = convoy.fleet.initialize(self.config, fs_context=True)
|
||||
self._set_clients(*clients)
|
||||
self.resource_client, self.compute_client, self.network_client, \
|
||||
_, _ = convoy.clients.create_arm_clients(self)
|
||||
self._cleanup_after_initialize(
|
||||
skip_global_config=False, skip_pool_config=True,
|
||||
skip_fs_config=False)
|
||||
clients = convoy.fleet.initialize(self.config, fs_context=True)
|
||||
|
||||
def initialize_for_keyvault(self):
|
||||
# type: (CliContext) -> None
|
||||
|
|
@ -104,7 +104,7 @@ class CliContext(object):
|
|||
"""
|
||||
self._read_credentials_config()
|
||||
self._set_global_cli_options()
|
||||
self.keyvault_client = convoy.fleet.create_keyvault_client(self)
|
||||
self.keyvault_client = convoy.clients.create_keyvault_client(self)
|
||||
self._init_config(
|
||||
skip_global_config=True, skip_pool_config=True,
|
||||
skip_fs_config=True)
|
||||
|
|
@ -112,19 +112,23 @@ class CliContext(object):
|
|||
skip_global_config=True, skip_pool_config=True,
|
||||
skip_fs_config=True)
|
||||
|
||||
def initialize_for_batch(self):
|
||||
# type: (CliContext) -> None
|
||||
def initialize_for_batch(self, init_clients_for_vnet=False):
|
||||
# type: (CliContext, bool) -> None
|
||||
"""Initialize context for batch commands
|
||||
:param CliContext self: this
|
||||
:param bool init_clients_for_vnet: intialize clients for vnet
|
||||
"""
|
||||
self._read_credentials_config()
|
||||
self._set_global_cli_options()
|
||||
self.keyvault_client = convoy.fleet.create_keyvault_client(self)
|
||||
self.keyvault_client = convoy.clients.create_keyvault_client(self)
|
||||
self._init_config(
|
||||
skip_global_config=False, skip_pool_config=False,
|
||||
skip_fs_config=True)
|
||||
clients = convoy.fleet.initialize(self)
|
||||
self._set_clients(*clients)
|
||||
self.resource_client, self.compute_client, self.network_client, \
|
||||
self.batch_mgmt_client, self.batch_client = \
|
||||
convoy.clients.create_arm_clients(self, batch_clients=True)
|
||||
self.blob_client, self.queue_client, self.table_client = \
|
||||
convoy.clients.create_storage_clients()
|
||||
self._cleanup_after_initialize(
|
||||
skip_global_config=False, skip_pool_config=False,
|
||||
skip_fs_config=True)
|
||||
|
|
@ -134,8 +138,17 @@ class CliContext(object):
|
|||
"""Initialize context for storage commands
|
||||
:param CliContext self: this
|
||||
"""
|
||||
# path is identical to batch
|
||||
self.initialize_for_batch()
|
||||
self._read_credentials_config()
|
||||
self._set_global_cli_options()
|
||||
self.keyvault_client = convoy.clients.create_keyvault_client(self)
|
||||
self._init_config(
|
||||
skip_global_config=False, skip_pool_config=False,
|
||||
skip_fs_config=True)
|
||||
self.blob_client, self.queue_client, self.table_client = \
|
||||
convoy.clients.create_storage_clients()
|
||||
self._cleanup_after_initialize(
|
||||
skip_global_config=False, skip_pool_config=False,
|
||||
skip_fs_config=True)
|
||||
|
||||
def _set_global_cli_options(self):
|
||||
# type: (CliContext) -> None
|
||||
|
|
@ -310,12 +323,19 @@ class CliContext(object):
|
|||
self.json_jobs = pathlib.Path(self.json_jobs)
|
||||
if self.json_jobs.exists():
|
||||
self._read_json_file(self.json_jobs)
|
||||
# adjust settings
|
||||
if not skip_fs_config:
|
||||
convoy.fleet.adjust_general_settings(self.config)
|
||||
convoy.fleet.populate_global_settings(self.config, not skip_fs_config)
|
||||
# show config if specified
|
||||
if self.show_config:
|
||||
logger.debug('config:\n' + json.dumps(self.config, indent=4))
|
||||
|
||||
def _set_clients(
|
||||
self, batch_client, blob_client, queue_client, table_client):
|
||||
self, batch_mgmt_client, batch_client, blob_client, queue_client,
|
||||
table_client):
|
||||
"""Sets clients for the context"""
|
||||
self.batch_mgmt_client = batch_mgmt_client
|
||||
self.batch_client = batch_client
|
||||
self.blob_client = blob_client
|
||||
self.queue_client = queue_client
|
||||
|
|
@ -495,16 +515,16 @@ def _aad_endpoint_option(f):
|
|||
callback=callback)(f)
|
||||
|
||||
|
||||
def _azure_management_subscription_id_option(f):
|
||||
def _azure_subscription_id_option(f):
|
||||
def callback(ctx, param, value):
|
||||
clictx = ctx.ensure_object(CliContext)
|
||||
clictx.subscription_id = value
|
||||
return value
|
||||
return click.option(
|
||||
'--management-subscription-id',
|
||||
'--subscription-id',
|
||||
expose_value=False,
|
||||
envvar='SHIPYARD_MANAGEMENT_SUBSCRIPTION_ID',
|
||||
help='Azure Management Subscription Id',
|
||||
envvar='SHIPYARD_SUBSCRIPTION_ID',
|
||||
help='Azure Subscription ID',
|
||||
callback=callback)(f)
|
||||
|
||||
|
||||
|
|
@ -612,6 +632,7 @@ def aad_options(f):
|
|||
|
||||
|
||||
def batch_options(f):
|
||||
f = _azure_subscription_id_option(f)
|
||||
f = _jobs_option(f)
|
||||
f = _pool_option(f)
|
||||
return f
|
||||
|
|
@ -624,7 +645,7 @@ def keyvault_options(f):
|
|||
|
||||
|
||||
def fs_options(f):
|
||||
f = _azure_management_subscription_id_option(f)
|
||||
f = _azure_subscription_id_option(f)
|
||||
f = _fs_option(f)
|
||||
return f
|
||||
|
||||
|
|
@ -970,10 +991,11 @@ def pool_listskus(ctx):
|
|||
@pass_cli_context
|
||||
def pool_add(ctx):
|
||||
"""Add a pool to the Batch account"""
|
||||
ctx.initialize_for_batch()
|
||||
ctx.initialize_for_batch(init_clients_for_vnet=True)
|
||||
convoy.fleet.action_pool_add(
|
||||
ctx.batch_client, ctx.blob_client, ctx.queue_client,
|
||||
ctx.table_client, ctx.config)
|
||||
ctx.resource_client, ctx.network_client, ctx.batch_mgmt_client,
|
||||
ctx.batch_client, ctx.blob_client, ctx.queue_client, ctx.table_client,
|
||||
ctx.config)
|
||||
|
||||
|
||||
@pool.command('list')
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче