diff --git a/convoy/batch.py b/convoy/batch.py index 6eb98e4..365ab15 100644 --- a/convoy/batch.py +++ b/convoy/batch.py @@ -919,9 +919,9 @@ def check_pool_nodes_runnable(batch_client, config): return False -def create_pool(batch_client, blob_client, config, pool): +def create_pool(batch_client, blob_client, config, pool, no_wait): # type: (batch.BatchServiceClient, azure.storage.blob.BlockBlobService, - # dict, batchmodels.PoolAddParameter) -> + # dict, batchmodels.PoolAddParameter, bool) -> # List[batchmodels.ComputeNode] """Create pool if not exists :param batch_client: The batch client to use. @@ -929,6 +929,7 @@ def create_pool(batch_client, blob_client, config, pool): :param azure.storage.blob.BlockBlobService blob_client: blob client :param dict config: configuration dict :param batchmodels.PoolAddParameter pool: pool addparameter object + :param bool no_Wait: do not wait for nodes to provision :rtype: list :return: list of nodes """ @@ -952,8 +953,11 @@ def create_pool(batch_client, blob_client, config, pool): sys.exit(1) else: logger.error('Pool {!r} already exists'.format(pool.id)) - # wait for pool idle - return wait_for_pool_ready(batch_client, blob_client, config, pool.id) + # wait for pool idle, if wait + if no_wait: + return None + else: + return wait_for_pool_ready(batch_client, blob_client, config, pool.id) def _add_admin_user_to_compute_node( diff --git a/convoy/fleet.py b/convoy/fleet.py index 267ca11..745b671 100644 --- a/convoy/fleet.py +++ b/convoy/fleet.py @@ -1812,14 +1812,14 @@ def _construct_auto_pool_specification( def _add_pool( resource_client, compute_client, network_client, batch_mgmt_client, - batch_client, blob_client, keyvault_client, config): + batch_client, blob_client, keyvault_client, config, no_wait): # type: (azure.mgmt.resource.resources.ResourceManagementClient, # azure.mgmt.compute.ComputeManagementClient, # azure.mgmt.network.NetworkManagementClient, # azure.mgmt.batch.BatchManagementClient, # azure.batch.batch_service_client.BatchServiceClient, # azure.storage.blob.BlockBlobService, - # azure.keyvault.KeyVaultClient, dict) -> None + # azure.keyvault.KeyVaultClient, dict, bool) -> None """Add a Batch pool to account :param azure.mgmt.resource.resources.ResourceManagementClient resource_client: resource client @@ -1833,11 +1833,31 @@ def _add_pool( :param azure.storage.blob.BlockBlobService blob_client: blob client :param azure.keyvault.KeyVaultClient keyvault_client: keyvault client :param dict config: configuration dict + :param bool no_wait: do not wait for nodes to provision """ # upload resource files and construct pool add parameter object pool_settings, gluster_on_compute, pool = _construct_pool_object( resource_client, compute_client, network_client, batch_mgmt_client, batch_client, blob_client, keyvault_client, config) + # check no wait settings + if no_wait: + if gluster_on_compute: + logger.error('forcing wait on pool due to glusterfs on compute') + no_wait = False + if pool_settings.transfer_files_on_pool_creation: + logger.error( + 'forcing wait on pool due to file transfer on pool creation') + no_wait = False + if settings.is_windows_pool(config): + if util.is_not_empty(pool_settings.rdp.username): + logger.warning( + 'skipping adding RDP user on Windows pool due to ' + 'disabling waiting for node provisioning') + else: + if util.is_not_empty(pool_settings.ssh.username): + logger.warning( + 'skipping adding SSH user on Linux pool due to ' + 'disabling waiting for node provisioning') # ingress data to Azure Blob Storage if specified storage_threads = [] if pool_settings.transfer_files_on_pool_creation: @@ -1845,7 +1865,11 @@ def _add_pool( batch_client, compute_client, network_client, config, rls=None, kind='storage') # create pool - nodes = batch.create_pool(batch_client, blob_client, config, pool) + nodes = batch.create_pool(batch_client, blob_client, config, pool, no_wait) + if no_wait: + logger.warning( + 'Not waiting for nodes to provision for pool: {}'.format(pool.id)) + return _pool = batch_client.pool.get(pool.id) pool_current_vm_count = ( _pool.current_dedicated_nodes + _pool.current_low_priority_nodes @@ -3358,7 +3382,7 @@ def action_cert_del(batch_client, config, sha1): def action_pool_add( resource_client, compute_client, network_client, batch_mgmt_client, batch_client, blob_client, table_client, keyvault_client, config, - recreate): + recreate, no_wait): # type: (azure.mgmt.resource.resources.ResourceManagementClient, # azure.mgmt.compute.ComputeManagementClient, # azure.mgmt.network.NetworkManagementClient, @@ -3366,7 +3390,7 @@ def action_pool_add( # azure.batch.batch_service_client.BatchServiceClient, # azure.storage.blob.BlockBlobService, # azure.cosmosdb.table.TableService, - # azure.keyvault.KeyVaultClient, dict, bool) -> None + # azure.keyvault.KeyVaultClient, dict, bool, bool) -> None """Action: Pool Add :param azure.mgmt.resource.resources.ResourceManagementClient resource_client: resource client @@ -3382,6 +3406,7 @@ def action_pool_add( :param azure.keyvault.KeyVaultClient keyvault_client: keyvault client :param dict config: configuration dict :param bool recreate: recreate + :param bool no_wait: do not wait for nodes to provision """ _check_batch_client(batch_client) # first check if pool exists to prevent accidential metadata clear or @@ -3415,7 +3440,7 @@ def action_pool_add( blob_client, table_client, config) _add_pool( resource_client, compute_client, network_client, batch_mgmt_client, - batch_client, blob_client, keyvault_client, config + batch_client, blob_client, keyvault_client, config, no_wait ) diff --git a/docs/20-batch-shipyard-usage.md b/docs/20-batch-shipyard-usage.md index 3713f32..a27a0c7 100644 --- a/docs/20-batch-shipyard-usage.md +++ b/docs/20-batch-shipyard-usage.md @@ -843,6 +843,9 @@ The `pool user` sub-command has the following sub-sub-commands: * `add` will add the pool defined in the pool configuration file to the Batch account + * `--no-wait` will not wait for nodes to provision successfully. This will + prevent creation of remote users and is incompatible with certain + options that require nodes to be provisioned. * `--recreate` will delete and recreate the pool if there already exists a pool with the same id. Note that you should only use this option if you are certain that it will not cause side-effects. diff --git a/shipyard.py b/shipyard.py index 27e50bf..15f6343 100755 --- a/shipyard.py +++ b/shipyard.py @@ -1529,19 +1529,21 @@ def pool(ctx): @pool.command('add') @click.option( '--recreate', is_flag=True, help='Recreate pool if it exists') +@click.option( + '--no-wait', is_flag=True, help='Do not wait for nodes to provision') @common_options @fs_option @batch_options @keyvault_options @aad_options @pass_cli_context -def pool_add(ctx, recreate): +def pool_add(ctx, recreate, no_wait): """Add a pool to the Batch account""" ctx.initialize_for_batch() convoy.fleet.action_pool_add( ctx.resource_client, ctx.compute_client, ctx.network_client, ctx.batch_mgmt_client, ctx.batch_client, ctx.blob_client, - ctx.table_client, ctx.keyvault_client, ctx.config, recreate) + ctx.table_client, ctx.keyvault_client, ctx.config, recreate, no_wait) @pool.command('exists') @@ -3039,7 +3041,7 @@ def slurm_cluster_orchestrate(ctx, storage_cluster_id): convoy.fleet.action_pool_add( ctx.resource_client, ctx.compute_client, ctx.network_client, ctx.batch_mgmt_client, ctx.batch_client, ctx.blob_client, - ctx.table_client, ctx.keyvault_client, ctx.config, False) + ctx.table_client, ctx.keyvault_client, ctx.config, False, False) convoy.fleet.action_slurm_cluster_create( ctx.auth_client, ctx.resource_client, ctx.compute_client, ctx.network_client, ctx.blob_client, ctx.table_client,