Update to Batch 7.0.0 SDK
- Breaking change: pool listskus -> account images - Support setting working directory for native mode - Resolves #286
This commit is contained in:
Fred Park
Родитель
b93f60213d
Коммит
4b9a004f1a
|
|
@ -34,8 +34,8 @@ import os
|
|||
import pickle
|
||||
import time
|
||||
# non-stdlib imports
|
||||
import azure.batch
|
||||
import azure.batch.models as batchmodels
|
||||
import azure.batch.batch_service_client as batch
|
||||
import msrest.authentication
|
||||
|
||||
# create logger
|
||||
|
|
@ -97,9 +97,9 @@ class TokenAuthentication(msrest.authentication.Authentication):
|
|||
|
||||
|
||||
def _create_credentials():
|
||||
# type: (None) -> azure.batch.batch_service_client.BatchServiceClient
|
||||
# type: (None) -> azure.batch.BatchServiceClient
|
||||
"""Create authenticated client
|
||||
:rtype: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:rtype: `azure.batch.BatchServiceClient`
|
||||
:return: batch_client
|
||||
"""
|
||||
# get the AAD token provided to the job manager
|
||||
|
|
@ -108,7 +108,7 @@ def _create_credentials():
|
|||
logger.debug('creating batch client for account url: {}'.format(
|
||||
account_service_url))
|
||||
credentials = TokenAuthentication(aad_token)
|
||||
batch_client = batch.BatchServiceClient(
|
||||
batch_client = azure.batch.BatchServiceClient(
|
||||
credentials, batch_url=account_service_url)
|
||||
batch_client.config.add_user_agent('batch-shipyard/rjm')
|
||||
return batch_client
|
||||
|
|
@ -119,7 +119,7 @@ def _submit_task_sub_collection(
|
|||
# type: (batch.BatchServiceClient, str, int, int, int, list, dict) -> None
|
||||
"""Submits a sub-collection of tasks, do not call directly
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:type batch_client: `azure.batch.BatchServiceClient`
|
||||
:param str job_id: job to add to
|
||||
:param int start: start offset, includsive
|
||||
:param int end: end offset, exclusive
|
||||
|
|
@ -188,7 +188,7 @@ def _add_task_collection(batch_client, job_id, task_map):
|
|||
# type: (batch.BatchServiceClient, str, dict) -> None
|
||||
"""Add a collection of tasks to a job
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:type batch_client: `azure.batch.BatchServiceClient`
|
||||
:param str job_id: job to add to
|
||||
:param dict task_map: task collection map to add
|
||||
"""
|
||||
|
|
@ -211,7 +211,7 @@ def _monitor_tasks(batch_client, job_id, numtasks):
|
|||
# type: (batch.BatchServiceClient, str, int) -> None
|
||||
"""Monitor tasks for completion
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:type batch_client: `azure.batch.BatchServiceClient`
|
||||
:param str job_id: job to add to
|
||||
:param int numtasks: number of tasks
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
azure-batch==6.0.1
|
||||
azure-batch==7.0.0
|
||||
msrest==0.5.5
|
||||
requests==2.21.0
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ import multiprocessing
|
|||
import os
|
||||
import pathlib
|
||||
# non-stdlib imports
|
||||
import azure.batch
|
||||
import azure.batch.batch_auth as batchauth
|
||||
import azure.batch.batch_service_client as batch
|
||||
|
||||
# create logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -55,13 +55,13 @@ def _setup_logger() -> None:
|
|||
|
||||
|
||||
def _create_credentials():
|
||||
# type: (None) -> azure.batch.batch_service_client.BatchServiceClient
|
||||
# type: (None) -> azure.batch.BatchServiceClient
|
||||
"""Create authenticated client
|
||||
:rtype: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:rtype: `azure.batch.BatchServiceClient`
|
||||
:return: batch_client
|
||||
"""
|
||||
ba, url, bakey = os.environ['SHIPYARD_BATCH_ENV'].split(';')
|
||||
batch_client = batch.BatchServiceClient(
|
||||
batch_client = azure.batch.BatchServiceClient(
|
||||
batchauth.SharedKeyCredentials(ba, bakey), batch_url=url)
|
||||
batch_client.config.add_user_agent('batch-shipyard/tfm')
|
||||
return batch_client
|
||||
|
|
@ -72,7 +72,7 @@ def _get_task_file(batch_client, job_id, task_id, filename, fp):
|
|||
# pathlib.Path) -> None
|
||||
"""Get a files from a task
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:type batch_client: `azure.batch.BatchServiceClient`
|
||||
:param str job_id: job id
|
||||
:param str task_id: task id
|
||||
:param str filename: file name
|
||||
|
|
@ -88,7 +88,7 @@ def get_all_files_via_task(batch_client, job_id, task_id, incl, excl, dst):
|
|||
# type: (batch.BatchServiceClient, str, str, list, list, str) -> None
|
||||
"""Get all files from a task
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:type batch_client: `azure.batch.BatchServiceClient`
|
||||
"""
|
||||
# prepare incl/excl filters
|
||||
if incl is not None:
|
||||
|
|
|
|||
|
|
@ -271,22 +271,91 @@ def log_batch_account_service_quota(batch_mgmt_client, config, location):
|
|||
logger.info(os.linesep.join(log))
|
||||
|
||||
|
||||
def list_node_agent_skus(batch_client, config):
|
||||
# type: (batch.BatchServiceClient, dict) -> None
|
||||
"""List all node agent skus
|
||||
def list_supported_images(
|
||||
batch_client, config, show_unrelated=False, show_unverified=False):
|
||||
# type: (batch.BatchServiceClient, dict, bool, bool) -> None
|
||||
"""List all supported images for the account
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.batch_service_client.BatchServiceClient`
|
||||
:type batch_client: `azure.batch.BatchServiceClient`
|
||||
:param dict config: configuration dict
|
||||
:param bool show_unrelated: show unrelated
|
||||
:param bool show_unverified: show unverified images
|
||||
"""
|
||||
if show_unverified:
|
||||
args = []
|
||||
else:
|
||||
args = [batchmodels.AccountListSupportedImagesOptions(
|
||||
filter='verificationType eq \'verified\'')]
|
||||
if settings.raw(config):
|
||||
util.print_raw_paged_output(batch_client.account.list_node_agent_skus)
|
||||
util.print_raw_paged_output(
|
||||
batch_client.account.list_supported_images, *args)
|
||||
return
|
||||
node_agent_skus = batch_client.account.list_node_agent_skus()
|
||||
for sku in node_agent_skus:
|
||||
for img in sku.verified_image_references:
|
||||
logger.info(
|
||||
'os_type={} publisher={} offer={} sku={} node_agent={}'.format(
|
||||
sku.os_type, img.publisher, img.offer, img.sku, sku.id))
|
||||
images = batch_client.account.list_supported_images(*args)
|
||||
image_map = {}
|
||||
for image in images:
|
||||
os_type = image.os_type.value
|
||||
if os_type not in image_map:
|
||||
image_map[os_type] = {}
|
||||
if (not show_unrelated and
|
||||
image.image_reference.publisher.lower() not in
|
||||
settings.get_valid_publishers()):
|
||||
continue
|
||||
if image.image_reference.publisher not in image_map[os_type]:
|
||||
image_map[os_type][image.image_reference.publisher] = {}
|
||||
if (image.image_reference.offer not in
|
||||
image_map[os_type][image.image_reference.publisher]):
|
||||
image_map[os_type][image.image_reference.publisher][
|
||||
image.image_reference.offer] = []
|
||||
image_map[os_type][image.image_reference.publisher][
|
||||
image.image_reference.offer].append({
|
||||
'sku': image.image_reference.sku,
|
||||
'na_sku': image.node_agent_sku_id,
|
||||
'verification': image.verification_type,
|
||||
'capabilities': image.capabilities,
|
||||
'support_eol': image.batch_support_end_of_life,
|
||||
})
|
||||
log = ['supported images (include unrelated={}, '
|
||||
'include unverified={})'.format(show_unrelated, show_unverified)]
|
||||
for os_type in image_map:
|
||||
log.append('* os type: {}'.format(os_type))
|
||||
for publisher in image_map[os_type]:
|
||||
log.append(' * publisher: {}'.format(publisher))
|
||||
for offer in image_map[os_type][publisher]:
|
||||
log.append(' * offer: {}'.format(offer))
|
||||
for image in image_map[os_type][publisher][offer]:
|
||||
log.append(' * sku: {}'.format(image['sku']))
|
||||
if util.is_not_empty(image['capabilities']):
|
||||
log.append(' * capabilities: {}'.format(
|
||||
','.join(image['capabilities'])))
|
||||
log.append(' * verification: {}'.format(
|
||||
image['verification']))
|
||||
if image['support_eol'] is not None:
|
||||
log.append(' * batch support eol: {}'.format(
|
||||
image['support_eol'].strftime("%Y-%m-%d")))
|
||||
log.append(' * node agent sku id: {}'.format(
|
||||
image['na_sku']))
|
||||
logger.info(os.linesep.join(log))
|
||||
|
||||
|
||||
def get_node_agent_for_image(batch_client, config, publisher, offer, sku):
|
||||
# type: (batch.BatchServiceClient, dict, str, str, str) -> tuple
|
||||
"""Get node agent for image
|
||||
:param batch_client: The batch client to use.
|
||||
:type batch_client: `azure.batch.BatchServiceClient`
|
||||
:param dict config: configuration dict
|
||||
:param str publisher: publisher
|
||||
:param str offer: offer
|
||||
:param str sku: sku
|
||||
:rtype: tuple
|
||||
:return: image ref and node agent sku id
|
||||
"""
|
||||
images = batch_client.account.list_supported_images()
|
||||
for image in images:
|
||||
if (image.image_reference.publisher.lower() == publisher.lower() and
|
||||
image.image_reference.offer.lower() == offer.lower() and
|
||||
image.image_reference.sku.lower() == sku.lower()):
|
||||
return image.image_reference, image.node_agent_sku_id
|
||||
return None, None
|
||||
|
||||
|
||||
def add_certificate_to_account(
|
||||
|
|
@ -4607,7 +4676,9 @@ def _construct_task(
|
|||
if native:
|
||||
batchtask.container_settings = batchmodels.TaskContainerSettings(
|
||||
container_run_options=' '.join(task.run_options),
|
||||
image_name=task.docker_image)
|
||||
image_name=task.docker_image,
|
||||
working_directory=task.working_dir,
|
||||
)
|
||||
# add additional resource files
|
||||
if util.is_not_empty(task.resource_files):
|
||||
for rf in task.resource_files:
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ from builtins import ( # noqa
|
|||
# stdlib imports
|
||||
import logging
|
||||
# non-stdlib imports
|
||||
import azure.batch
|
||||
import azure.batch.batch_auth as batchauth
|
||||
import azure.batch.batch_service_client as batchsc
|
||||
import azure.cosmosdb.table as azuretable
|
||||
import azure.keyvault
|
||||
import azure.mgmt.authorization
|
||||
|
|
@ -247,7 +247,7 @@ def create_all_clients(ctx, batch_clients=False):
|
|||
# azure.mgmt.network.NetworkManagementClient,
|
||||
# azure.mgmt.storage.StorageManagementClient,
|
||||
# azure.mgmt.batch.BatchManagementClient,
|
||||
# azure.batch.batch_service_client.BatchServiceClient]
|
||||
# azure.batch.BatchServiceClient]
|
||||
"""Create all arm clients and batch service client
|
||||
:param CliContext ctx: Cli Context
|
||||
:param bool batch_clients: create batch clients
|
||||
|
|
@ -259,7 +259,7 @@ def create_all_clients(ctx, batch_clients=False):
|
|||
azure.mgmt.network.NetworkManagementClient,
|
||||
azure.mgmt.storage.StorageManagementClient,
|
||||
azure.mgmt.batch.BatchManagementClient,
|
||||
azure.batch.batch_service_client.BatchServiceClient)
|
||||
azure.batch.BatchServiceClient)
|
||||
"""
|
||||
mgmt = settings.credentials_management(ctx.config)
|
||||
subscription_id = ctx.subscription_id or mgmt.subscription_id
|
||||
|
|
@ -333,10 +333,10 @@ def create_keyvault_client(ctx):
|
|||
|
||||
|
||||
def _create_batch_service_client(ctx):
|
||||
# type: (CliContext) -> azure.batch.batch_service_client.BatchServiceClient
|
||||
# type: (CliContext) -> azure.batch.BatchServiceClient
|
||||
"""Create batch service client
|
||||
:param CliContext ctx: Cli Context
|
||||
:rtype: azure.batch.batch_service_client.BatchServiceClient
|
||||
:rtype: azure.batch.BatchServiceClient
|
||||
:return: batch service client
|
||||
"""
|
||||
bc = settings.credentials_batch(ctx.config)
|
||||
|
|
@ -348,7 +348,7 @@ def _create_batch_service_client(ctx):
|
|||
else:
|
||||
credentials = batchauth.SharedKeyCredentials(
|
||||
bc.account, bc.account_key)
|
||||
batch_client = batchsc.BatchServiceClient(
|
||||
batch_client = azure.batch.BatchServiceClient(
|
||||
credentials, batch_url=bc.account_service_url)
|
||||
_modify_client_for_retry_and_user_agent(batch_client)
|
||||
return batch_client
|
||||
|
|
|
|||
|
|
@ -897,54 +897,21 @@ def _pick_node_agent_for_vm(batch_client, config, pool_settings):
|
|||
publisher = pool_settings.vm_configuration.publisher
|
||||
offer = pool_settings.vm_configuration.offer
|
||||
sku = pool_settings.vm_configuration.sku
|
||||
# backward compat for CentOS HPC 7.1, 7.3, 7.4 and normal 7.4, 7.5, 7.6
|
||||
if publisher == 'openlogic':
|
||||
if ((offer == 'centos-hpc' and
|
||||
(sku == '7.1' or sku == '7.3' or sku == '7.4')) or
|
||||
(offer == 'centos' and
|
||||
(sku == '7.4' or sku == '7.5' or sku == '7.6'))):
|
||||
return ({
|
||||
'publisher': publisher,
|
||||
'offer': offer,
|
||||
'sku': sku,
|
||||
'version': pool_settings.vm_configuration.version,
|
||||
}, 'batch.node.centos 7')
|
||||
# support windows server semi annual
|
||||
if (publisher == 'microsoftwindowsserver' and
|
||||
offer == 'windowsserversemiannual' and 'with-containers' in sku):
|
||||
return ({
|
||||
'publisher': publisher,
|
||||
'offer': offer,
|
||||
'sku': sku,
|
||||
'version': pool_settings.vm_configuration.version,
|
||||
}, 'batch.node.windows amd64')
|
||||
# pick latest sku
|
||||
node_agent_skus = batch_client.account.list_node_agent_skus()
|
||||
skus_to_use = [
|
||||
(nas, image_ref) for nas in node_agent_skus
|
||||
for image_ref in sorted(
|
||||
nas.verified_image_references,
|
||||
key=lambda item: item.sku
|
||||
)
|
||||
if image_ref.publisher.lower() == publisher and
|
||||
image_ref.offer.lower() == offer and
|
||||
image_ref.sku.lower() == sku
|
||||
]
|
||||
try:
|
||||
sku_to_use, image_ref_to_use = skus_to_use[-1]
|
||||
except IndexError:
|
||||
image_ref, node_agent = batch.get_node_agent_for_image(
|
||||
batch_client, config, publisher, offer, sku)
|
||||
if image_ref is None:
|
||||
raise RuntimeError(
|
||||
('Could not find an Azure Batch Node Agent Sku for this '
|
||||
'offer={} publisher={} sku={}. You can list the valid and '
|
||||
'available Marketplace images with the command: pool '
|
||||
'listskus').format(
|
||||
'available Marketplace images with the command: account '
|
||||
'images').format(
|
||||
pool_settings.vm_configuration.offer,
|
||||
pool_settings.vm_configuration.publisher,
|
||||
pool_settings.vm_configuration.sku))
|
||||
# set image version to use
|
||||
image_ref_to_use.version = pool_settings.vm_configuration.version
|
||||
logger.info('deploying vm config: {}'.format(image_ref_to_use))
|
||||
return (image_ref_to_use, sku_to_use.id)
|
||||
image_ref.version = pool_settings.vm_configuration.version
|
||||
logger.info('deploying vm config: {}'.format(image_ref))
|
||||
return (image_ref, node_agent)
|
||||
|
||||
|
||||
def _check_for_batch_aad(bc, rmsg):
|
||||
|
|
@ -2726,6 +2693,25 @@ def _check_batch_client(batch_client):
|
|||
'proper "batch" credentials')
|
||||
|
||||
|
||||
def action_account_images(
|
||||
batch_client, config, show_unrelated, show_unverified):
|
||||
# type: (batchsc.BatchServiceClient, dict, bool, bool) -> None
|
||||
"""Action: Account Images
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
:param bool show_unrelated: show unrelated
|
||||
:param bool show_unverified: show unverified
|
||||
"""
|
||||
_check_batch_client(batch_client)
|
||||
if settings.raw(config) and not show_unrelated:
|
||||
logger.warning('force enabling --show-unrelated with --raw')
|
||||
show_unrelated = True
|
||||
batch.list_supported_images(
|
||||
batch_client, config, show_unrelated=show_unrelated,
|
||||
show_unverified=show_unverified)
|
||||
|
||||
|
||||
def action_account_info(batch_mgmt_client, config, name, resource_group):
|
||||
# type: (azure.mgmt.batch.BatchManagementClient, dict, str, str) -> None
|
||||
"""Action: Account Info
|
||||
|
|
@ -3123,17 +3109,6 @@ def action_cert_del(batch_client, config, sha1):
|
|||
batch.del_certificate_from_account(batch_client, config, sha1)
|
||||
|
||||
|
||||
def action_pool_listskus(batch_client, config):
|
||||
# type: (batchsc.BatchServiceClient, dict) -> None
|
||||
"""Action: Pool Listskus
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
_check_batch_client(batch_client)
|
||||
batch.list_node_agent_skus(batch_client, config)
|
||||
|
||||
|
||||
def action_pool_add(
|
||||
resource_client, compute_client, network_client, batch_mgmt_client,
|
||||
batch_client, blob_client, table_client, keyvault_client, config,
|
||||
|
|
|
|||
|
|
@ -62,8 +62,7 @@ _TENSORBOARD_LOG_ARGS = frozenset((
|
|||
def tunnel_tensorboard(batch_client, config, jobid, taskid, logdir, image):
|
||||
# type: (batchsc.BatchServiceClient, dict, str, str, str, str) -> None
|
||||
"""Create an SSH tunnel for Tensorboard running on compute nodes
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.batch.BatchServiceClient batch_client: batch client
|
||||
:param dict config: configuration dict
|
||||
:param str jobid: job id to list
|
||||
:param str taskid: task id to list
|
||||
|
|
@ -251,8 +250,7 @@ def tunnel_tensorboard(batch_client, config, jobid, taskid, logdir, image):
|
|||
def mirror_batch_shipyard_images(batch_client, config, script):
|
||||
# type: (batchsc.BatchServiceClient, dict, pathlib.Path) -> None
|
||||
"""Mirror Batch Shipyard images to a fallback registry
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.batch.BatchServiceClient batch_client: batch client
|
||||
:param dict config: configuration dict
|
||||
:param pathlib.Path script: script to mirror
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -130,6 +130,10 @@ _VM_IB_CLASS = {
|
|||
r'^standard_(((h|nc|nd)+[\d]+m?rs?(_v[\d])?))$', re.IGNORECASE),
|
||||
'edr_ib': re.compile(r'^standard_(hc|hb)+[\d]+rs$', re.IGNORECASE),
|
||||
}
|
||||
_VALID_PUBLISHERS = frozenset((
|
||||
'canonical', 'credativ', 'microsoft-azure-batch',
|
||||
'microsoftwindowsserver', 'openlogic'
|
||||
))
|
||||
_SINGULARITY_COMMANDS = frozenset(('exec', 'run'))
|
||||
_FORBIDDEN_MERGE_TASK_PROPERTIES = frozenset((
|
||||
'depends_on', 'depends_on_range', 'multi_instance', 'task_factory'
|
||||
|
|
@ -342,6 +346,7 @@ TaskSettings = collections.namedtuple(
|
|||
'envfile', 'resource_files', 'command', 'infiniband', 'gpu',
|
||||
'depends_on', 'depends_on_range', 'max_task_retries', 'max_wall_time',
|
||||
'retention_time', 'multi_instance', 'default_exit_options',
|
||||
'working_dir',
|
||||
]
|
||||
)
|
||||
MultiInstanceSettings = collections.namedtuple(
|
||||
|
|
@ -588,6 +593,15 @@ def get_metadata_version_name():
|
|||
return _METADATA_VERSION_NAME
|
||||
|
||||
|
||||
def get_valid_publishers():
|
||||
# type: (None) -> str
|
||||
"""Get valid publishers
|
||||
:rtype: str
|
||||
:return: publisher set
|
||||
"""
|
||||
return _VALID_PUBLISHERS
|
||||
|
||||
|
||||
def get_tensorboard_docker_image():
|
||||
# type: (None) -> Tuple[str, str]
|
||||
"""Get tensorboard docker image
|
||||
|
|
@ -3793,6 +3807,23 @@ def task_settings(
|
|||
def_wd = '%AZ_BATCH_TASK_WORKING_DIR%'
|
||||
else:
|
||||
def_wd = '$AZ_BATCH_TASK_WORKING_DIR'
|
||||
# set working directory if not already set
|
||||
if def_wd != 'container':
|
||||
if util.is_not_empty(docker_image):
|
||||
if not any((x.startswith('-w ') or x.startswith('--workdir '))
|
||||
for x in run_opts):
|
||||
run_opts.append('-w {}'.format(def_wd))
|
||||
else:
|
||||
if not any(x.startswith('--pwd ') for x in run_opts):
|
||||
run_opts.append('--pwd {}'.format(def_wd))
|
||||
working_dir = (
|
||||
batchmodels.ContainerWorkingDirectory.task_working_directory
|
||||
)
|
||||
else:
|
||||
working_dir = (
|
||||
batchmodels.ContainerWorkingDirectory.container_image_default
|
||||
)
|
||||
del def_wd
|
||||
# bind root dir and set working dir
|
||||
if not native:
|
||||
restrict_bind = _kv_read(
|
||||
|
|
@ -3817,15 +3848,6 @@ def task_settings(
|
|||
run_opts.append(
|
||||
'{} $AZ_BATCH_NODE_ROOT_DIR:'
|
||||
'$AZ_BATCH_NODE_ROOT_DIR'.format(bindparm))
|
||||
# set working directory if not already set
|
||||
if def_wd != 'container':
|
||||
if util.is_not_empty(docker_image):
|
||||
if not any((x.startswith('-w ') or x.startswith('--workdir '))
|
||||
for x in run_opts):
|
||||
run_opts.append('-w {}'.format(def_wd))
|
||||
else:
|
||||
if not any(x.startswith('--pwd ') for x in run_opts):
|
||||
run_opts.append('--pwd {}'.format(def_wd))
|
||||
if util.is_not_empty(data_volumes):
|
||||
dv = global_resources_data_volumes(config)
|
||||
for dvkey in data_volumes:
|
||||
|
|
@ -4255,6 +4277,7 @@ def task_settings(
|
|||
name=name,
|
||||
run_options=run_opts,
|
||||
docker_exec_options=docker_exec_options,
|
||||
working_dir=working_dir,
|
||||
environment_variables=env_vars,
|
||||
environment_variables_keyvault_secret_id=ev_secid,
|
||||
envfile=envfile,
|
||||
|
|
|
|||
|
|
@ -192,7 +192,7 @@ custom image, please see the
|
|||
same subscription and region as the Batch account.
|
||||
* (required for custom image) `node_agent` is the node agent sku id to
|
||||
use with this custom image. You can view supported base images and
|
||||
their node agent sku ids with the `pool listskus` command.
|
||||
their node agent sku ids with the `account images` command.
|
||||
* (optional) `native` will opt to use native Docker container support
|
||||
if possible. This provides better task management (such as job and
|
||||
task termination while tasks are running), in exchange for some other
|
||||
|
|
|
|||
|
|
@ -547,9 +547,7 @@ directory for the container execution is not explicitly set. The default is
|
|||
directory, you can pass the appropriate working directory parameter to the
|
||||
container runtime through either `additional_docker_run_options` or
|
||||
`additional_singularity_options`. A working directory option specified within
|
||||
that property takes precedence over this option. Note that this option does
|
||||
not work in `native` mode currently; `native` mode will always override this
|
||||
option to `batch`.
|
||||
that property takes precedence over this option.
|
||||
* (optional) `restrict_default_bind_mounts` will restrict the mapped
|
||||
host directories into the container. If this property is set to `true`,
|
||||
only the `$AZ_BATCH_TASK_DIR` is mapped from the host into the container.
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ may change/break if the underlying service version changes. It is important
|
|||
to pin the Batch Shipyard release to a specific version if using this feature
|
||||
and perform upgrade testing/validation for your scenario and workflow between
|
||||
releases. The following commands support this option:
|
||||
* `account images`
|
||||
* `account info`
|
||||
* `account quota`
|
||||
* `cert list`
|
||||
|
|
@ -142,7 +143,6 @@ releases. The following commands support this option:
|
|||
* `pool images list`
|
||||
* `pool images update`
|
||||
* `pool list`
|
||||
* `pool listskus`
|
||||
* `pool nodes count`
|
||||
* `pool nodes grls`
|
||||
* `pool nodes list`
|
||||
|
|
@ -255,11 +255,17 @@ categories
|
|||
## `account` Command
|
||||
The `account` command has the following sub-commands:
|
||||
```
|
||||
info Retrieve Batch account information and quotas
|
||||
list Retrieve a list of Batch accounts and...
|
||||
quota Retrieve Batch account quota at the...
|
||||
images List available VM images available to the Batch account
|
||||
info Retrieve Batch account information and quotas
|
||||
list Retrieve a list of Batch accounts and associated quotas in...
|
||||
quota Retrieve Batch account quota at the subscription level for the...
|
||||
```
|
||||
|
||||
* `images` lists available VM images to deploy as compute nodes in a Batch
|
||||
pool
|
||||
* `--show-unrelated` will additionally list images that are unrelated for
|
||||
Batch Shipyard. This option is not applicable with `--raw`.
|
||||
* `--show-unverified` will additionally list images that are unverified
|
||||
* `info` provides information about the specified batch account provided
|
||||
in credentials
|
||||
* `--name` is the name of the Batch account to query instead of the
|
||||
|
|
@ -794,7 +800,6 @@ The `pool` command has the following sub-commands:
|
|||
exists Check if a pool exists
|
||||
images Container images actions
|
||||
list List all pools in the Batch account
|
||||
listskus List available VM configurations available to the Batch account
|
||||
nodes Compute node actions
|
||||
rdp Interactively login via RDP to a node in a pool
|
||||
resize Resize a pool
|
||||
|
|
|
|||
|
|
@ -238,9 +238,9 @@ following:
|
|||
* The `arm_image_id` points to a valid ARM Image resource
|
||||
* `node_agent` is populated with the correct node agent sku id which
|
||||
corresponds to the distribution used in the custom image. For instance,
|
||||
if your custom image is based on Ubuntu 16.04, you would use
|
||||
`batch.node.ubuntu 16.04` as the `node_agent` value. You can view a
|
||||
complete list of supported node agent sku ids with the `pool listskus`
|
||||
if your custom image is based on Ubuntu 18.04, you would use
|
||||
`batch.node.ubuntu 18.04` as the `node_agent` value. You can view a
|
||||
complete list of supported node agent sku ids with the `account images`
|
||||
command.
|
||||
|
||||
### ARM Image Retention Requirements
|
||||
|
|
|
|||
|
|
@ -155,11 +155,11 @@ If you are using a `platform_image`, you may encounter an error such as:
|
|||
```
|
||||
RuntimeError: Could not find an Azure Batch Node Agent Sku for this
|
||||
offer=abc publisher=def sku=xyz. You can list the valid and available
|
||||
Marketplace images with the command: pool listskus
|
||||
Marketplace images with the command: account images
|
||||
```
|
||||
|
||||
This problem can happen if you are specifying a `sku` that is not listed
|
||||
by the `pool listskus` command. You will need to update your `sku` field
|
||||
by the `account images` command. You will need to update your `sku` field
|
||||
to one that is listed.
|
||||
|
||||
## <a name="task"></a>Job/Task Execution Issues
|
||||
|
|
|
|||
|
|
@ -83,7 +83,6 @@ multi-instance, then it is strongly recommended to use `native` mode.
|
|||
Disadvantages of `native` mode are:
|
||||
|
||||
* Singularity containers are not supported.
|
||||
* `default_working_dir` in jobs cannot be changed from `batch`.
|
||||
* `input_data` of any kind at the task-level is not possible; you must either
|
||||
use `resource_files` or build your own solution.
|
||||
* `output_data` options are limited and egress to `azure_storage` Azure Files
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
azure-batch==6.0.1
|
||||
azure-batch==7.0.0
|
||||
azure-cosmosdb-table==1.0.5
|
||||
azure-mgmt-compute==4.4.0
|
||||
azure-mgmt-resource==2.1.0
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
azure-batch==6.0.1
|
||||
azure-batch==7.0.0
|
||||
azure-cosmosdb-table==1.0.5
|
||||
azure-mgmt-compute==4.4.0
|
||||
azure-mgmt-network==2.5.1
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
adal==1.2.1
|
||||
azure-batch==6.0.1
|
||||
azure-batch==7.0.0
|
||||
azure-cosmosdb-table==1.0.5
|
||||
azure-keyvault==1.1.0
|
||||
azure-mgmt-authorization==0.51.1
|
||||
|
|
|
|||
29
shipyard.py
29
shipyard.py
|
|
@ -1014,6 +1014,23 @@ def account(ctx):
|
|||
pass
|
||||
|
||||
|
||||
@account.command('images')
|
||||
@click.option(
|
||||
'--show-unrelated', is_flag=True, help='Include unrelated images')
|
||||
@click.option(
|
||||
'--show-unverified', is_flag=True, help='Include unverified images')
|
||||
@common_options
|
||||
@batch_options
|
||||
@keyvault_options
|
||||
@aad_options
|
||||
@pass_cli_context
|
||||
def account_images(ctx, show_unrelated, show_unverified):
|
||||
"""List available VM images available to the Batch account"""
|
||||
ctx.initialize_for_batch()
|
||||
convoy.fleet.action_account_images(
|
||||
ctx.batch_client, ctx.config, show_unrelated, show_unverified)
|
||||
|
||||
|
||||
@account.command('info')
|
||||
@click.option('--name', help='Batch account name')
|
||||
@click.option('--resource-group', help='Batch account resource group')
|
||||
|
|
@ -1510,18 +1527,6 @@ def pool(ctx):
|
|||
pass
|
||||
|
||||
|
||||
@pool.command('listskus')
|
||||
@common_options
|
||||
@batch_options
|
||||
@keyvault_options
|
||||
@aad_options
|
||||
@pass_cli_context
|
||||
def pool_listskus(ctx):
|
||||
"""List available VM configurations available to the Batch account"""
|
||||
ctx.initialize_for_batch()
|
||||
convoy.fleet.action_pool_listskus(ctx.batch_client, ctx.config)
|
||||
|
||||
|
||||
@pool.command('add')
|
||||
@click.option(
|
||||
'--recreate', is_flag=True, help='Recreate pool if it exists')
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
azure-batch==6.0.1
|
||||
azure-batch==7.0.0
|
||||
azure-cosmosdb-table==1.0.5
|
||||
azure-mgmt-storage==3.1.1
|
||||
azure-mgmt-resource==2.1.0
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче