Allow specifying the autoscratch task id

- Update docs regarding pool id naming requirements

config_templates/jobs.yaml

@@ -42,6 +42,7 @@ job_specifications:
   - joblevelsharedvol
   auto_scratch:
     setup: block
+    task_id: batch-shipyard-autoscratch
     num_instances: pool_current_dedicated
   job_preparation:
     command: myjpcommand

convoy/batch.py

@@ -70,7 +70,6 @@ _MAX_EXECUTOR_WORKERS = min((multiprocessing.cpu_count() * 4, 32))
 _MAX_REBOOT_RETRIES = 5
 _SSH_TUNNEL_SCRIPT = 'ssh_docker_tunnel_shipyard.sh'
 _TASKMAP_PICKLE_FILE = 'taskmap.pickle'
-_AUTOSCRATCH_TASK_ID = 'batch-shipyard-autoscratch'
 _RUN_ELEVATED = batchmodels.UserIdentity(
     auto_user=batchmodels.AutoUserSpecification(
         scope=batchmodels.AutoUserScope.pool,
@@ -4901,15 +4900,16 @@ def _construct_task(
         )
     # add autoscratch dependency
     if autoscratch_setup == 'dependency':
+        astaskid = settings.job_auto_scratch_task_id(jobspec)
         if batchtask.depends_on is None:
             batchtask.depends_on = batchmodels.TaskDependencies(
-                task_ids=[_AUTOSCRATCH_TASK_ID],
+                task_ids=[astaskid],
                 task_id_ranges=None,
             )
         elif batchtask.depends_on.task_ids is None:
-            batchtask.depends_on.task_ids = [_AUTOSCRATCH_TASK_ID]
+            batchtask.depends_on.task_ids = [astaskid]
         else:
-            batchtask.depends_on.task_ids.append(_AUTOSCRATCH_TASK_ID)
+            batchtask.depends_on.task_ids.append(astaskid)
     # add exit conditions
     if on_task_failure == batchmodels.OnTaskFailure.no_action:
         job_action = None
@@ -5008,7 +5008,7 @@ def _create_auto_scratch_volume(
             'Cannot create an auto_scratch volume with no current '
             'dedicated, low priority nodes or zero specified vm counts')
     batchtask = batchmodels.TaskAddParameter(
-        id=_AUTOSCRATCH_TASK_ID,
+        id=settings.job_auto_scratch_task_id(jobspec),
         multi_instance_settings=batchmodels.MultiInstanceSettings(
             number_of_instances=num_instances,
             coordination_command_line=util.wrap_commands_in_shell([
@@ -5154,6 +5154,7 @@ def add_jobs(
         # 4. if there are multi-instance tasks
         auto_complete = settings.job_auto_complete(jobspec)
         autoscratch_setup = settings.job_auto_scratch_setup(jobspec)
+        autoscratch_task_id = settings.job_auto_scratch_task_id(jobspec)
         autoscratch_task = None
         jobschedule = None
         multi_instance = False
@@ -5336,7 +5337,7 @@ def add_jobs(
             jrtaskcmd.append(
                 '$AZ_BATCH_NODE_ROOT_DIR/workitems/{}/job-1/{}/{} '
                 'stop {}'.format(
-                    job_id, _AUTOSCRATCH_TASK_ID, asfile[0], job_id)
+                    job_id, autoscratch_task_id, asfile[0], job_id)
             )
         if multi_instance and auto_complete and not native:
             jrtaskcmd.extend([
@@ -5620,7 +5621,7 @@ def add_jobs(
                         if autoscratch_avail and autoscratch_setup is not None:
                             try:
                                 autoscratch_task = batch_client.task.get(
-                                    job_id, _AUTOSCRATCH_TASK_ID)
+                                    job_id, autoscratch_task_id)
                                 if (autoscratch_task.execution_info is None and
                                         autoscratch_setup == 'block'):
                                     raise RuntimeError(

convoy/settings.py

@@ -3364,6 +3364,20 @@ def job_auto_scratch_setup(conf):
         _kv_read_checked(conf, 'auto_scratch', default={}), 'setup')
 
 
+def job_auto_scratch_task_id(conf):
+    # type: (dict) -> str
+    """Get job auto scratch task id setting
+    :param dict conf: job configuration object
+    :rtype: str
+    :return: job auto scratch task id
+    """
+    return _kv_read_checked(
+        _kv_read_checked(conf, 'auto_scratch', default={}),
+        'task_id',
+        default='batch-shipyard-autoscratch'
+    )
+
+
 def job_auto_scratch_num_instances(conf):
     # type: (dict) -> str
     """Get job auto scratch setting

docs/13-batch-shipyard-configuration-pool.md

@@ -156,7 +156,11 @@ pool_specification:
 
 The `pool_specification` property has the following members:
 
-* (required) `id` is the compute pool ID.
+* (required) `id` is the compute pool ID. This value can be any combination
+of alphanumeric characters including hyphens and underscores up to 64
+characters in length. If this pool specification is used for an `auto_pool`
+then the maximum length of this value is 20 characters which becomes the
+prefix for the autopool id.
 * (required) `vm_configuration` specifies the image configuration for the
 VM. Either `platform_image` or `custom_image` must be specified. You cannot
 specify both. Please see the

docs/14-batch-shipyard-configuration-jobs.md

@@ -68,6 +68,7 @@ job_specifications:
   - joblevelsharedvol
   auto_scratch:
     setup: block
+    task_id: batch-shipyard-autoscratch
     num_instances: pool_current_dedicated
   job_preparation:
     command: myjpcommand
@@ -509,6 +510,8 @@ corresponding pool configuration.
     the `block` client-side wait. Note that if `dependency` is specified as
     the setup method and the task fails, it will block all subsequent tasks
     in the job. There is no default value and an option must be specified.
+  * (optional) `task_id` is the task id of the autoscratch setup task. If
+    not specified, the default is `batch-shipyard-autoscratch`.
   * (required) `num_instances` is the number of instances for the auto scratch
     volume to span. Note that if you specify a number less than the number
     of available nodes, then subsequent tasks may not get assigned to nodes

schemas/jobs.yaml

@@ -110,6 +110,8 @@ mapping:
                 type: str
                 required: true
                 enum: ['block', 'dependency']
+              task_id:
+                type: str
               num_instances:
                 required: true
                 type: text