Post-release fixes

- Fix merge task with autogen task id
- Recipe updates
- Other minor fixes

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## [Unreleased]
 
+### Fixed
+- Fix merge task regression with enhanced autogenerated task id support
+
 ## [3.9.0] - 2019-11-15 (SC19 Edition)
 ### Added
 - Support for [Encrypted Singularity Containers](docs/50-batch-shipyard-encrypted-containers.md)

convoy/batch.py

@@ -5630,7 +5630,7 @@ def add_jobs(
         merge_task_id = None
         if has_merge_task:
             ntasks += 1
-            _task = settings.job_merge_task(jobspec)
+            _task = settings.job_merge_task(config, jobspec)
             existing_tasklist, merge_task_id, lasttaskic, gpu, ib = \
                 _construct_task(
                     batch_client, blob_client, keyvault_client, config,

convoy/settings.py

@@ -3545,14 +3545,23 @@ def job_has_merge_task(conf):
     return True
 
 
-def job_merge_task(conf):
+def job_merge_task(config, conf):
     # type: (dict) -> dict
     """Gets merge task
+    :param dict config: configuration object
     :param dict conf: job configuration object
     :rtype: dict
     :return: merge task
     """
-    return conf['merge_task']
+    if 'autogenerated_task_id' in conf:
+        prefix, padding = autogenerated_task_id_settings(conf, level='job')
+    else:
+        prefix, padding = autogenerated_task_id_settings(
+            config, level='global')
+    _task = conf['merge_task']
+    _task['##task_id_prefix'] = prefix
+    _task['##task_id_padding'] = padding
+    return _task
 
 
 def job_force_enable_task_dependencies(conf):

images/singularity/cli.def

@@ -1,7 +1,7 @@
 # Singularity for Azure/batch-shipyard (cli)
 
 Bootstrap: library
-From: alpine:3.10
+From: alpine:3.9
 
 %post
 apk update

recipes/BLAST-CPU/README.md

@@ -61,13 +61,12 @@ shipyard jobs add --jobs jobs-split.yaml --tail stdout.txt
 shipyard jobs add --jobs jobs-blast.yaml
 
 # poll the merge task until it completes
-shipyard jobs tasks list --jobid blast --taskid merge-task-00001 --poll-until-tasks-complete
+shipyard jobs tasks list --jobid blast --taskid merge-task-00000 --poll-until-tasks-complete
 
 # optionally egress the results.txt file from the compute node to local machine
-shipyard data files task --filespec blast,merge-task-00001,wd/results.txt
+shipyard data files task --filespec blast,merge-task-00000,wd/results.txt
 
 # clean up
-shipyard jobs del -y --jobs jobs-split.yaml
 shipyard jobs del -y --jobs jobs-blast.yaml
 shipyard pool del -y
 ```

recipes/HPLinpack-Infiniband-IntelMPI/config/singularity/jobs.yaml

@@ -18,6 +18,6 @@ job_specifications:
       mpi:
         runtime: intelmpi-ofa
         processes_per_node: 1
-      pre_execution_command: source setup_hplinpack.sh -a $AVX -n 50000; source /opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh
+      pre_execution_command: source setup_hplinpack.sh -a $AVX -n 50000; module load mpi/impi; source $MPI_BIN/mpivars.sh
     # need to source mpivars again as the container intel mpi/benchmark is not the exact version as on the host
     command: /bin/bash -c "source /opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh; cd /opt/intel/mkl/benchmarks/mp_linpack && ./runme_intel64_prv -p $P -q $Q -b $B $PSIZE"