From d14b8aa8b4d381955acd9b27be8fbbd6167f6e26 Mon Sep 17 00:00:00 2001 From: Fred Park Date: Sun, 17 Nov 2019 04:07:48 +0000 Subject: [PATCH] Post-release fixes - Fix merge task with autogen task id - Recipe updates - Other minor fixes --- CHANGELOG.md | 3 +++ convoy/batch.py | 2 +- convoy/settings.py | 13 +++++++++++-- images/singularity/cli.def | 2 +- recipes/BLAST-CPU/README.md | 5 ++--- .../config/singularity/jobs.yaml | 2 +- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcc3a66..7a35761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Fixed +- Fix merge task regression with enhanced autogenerated task id support + ## [3.9.0] - 2019-11-15 (SC19 Edition) ### Added - Support for [Encrypted Singularity Containers](docs/50-batch-shipyard-encrypted-containers.md) diff --git a/convoy/batch.py b/convoy/batch.py index 6f46c6c..6eb98e4 100644 --- a/convoy/batch.py +++ b/convoy/batch.py @@ -5630,7 +5630,7 @@ def add_jobs( merge_task_id = None if has_merge_task: ntasks += 1 - _task = settings.job_merge_task(jobspec) + _task = settings.job_merge_task(config, jobspec) existing_tasklist, merge_task_id, lasttaskic, gpu, ib = \ _construct_task( batch_client, blob_client, keyvault_client, config, diff --git a/convoy/settings.py b/convoy/settings.py index 226597c..1d6cb23 100644 --- a/convoy/settings.py +++ b/convoy/settings.py @@ -3545,14 +3545,23 @@ def job_has_merge_task(conf): return True -def job_merge_task(conf): +def job_merge_task(config, conf): # type: (dict) -> dict """Gets merge task + :param dict config: configuration object :param dict conf: job configuration object :rtype: dict :return: merge task """ - return conf['merge_task'] + if 'autogenerated_task_id' in conf: + prefix, padding = autogenerated_task_id_settings(conf, level='job') + else: + prefix, padding = autogenerated_task_id_settings( + config, level='global') + _task = conf['merge_task'] + _task['##task_id_prefix'] = prefix + _task['##task_id_padding'] = padding + return _task def job_force_enable_task_dependencies(conf): diff --git a/images/singularity/cli.def b/images/singularity/cli.def index eedd2a5..4949b75 100644 --- a/images/singularity/cli.def +++ b/images/singularity/cli.def @@ -1,7 +1,7 @@ # Singularity for Azure/batch-shipyard (cli) Bootstrap: library -From: alpine:3.10 +From: alpine:3.9 %post apk update diff --git a/recipes/BLAST-CPU/README.md b/recipes/BLAST-CPU/README.md index afdf52f..d70eaef 100644 --- a/recipes/BLAST-CPU/README.md +++ b/recipes/BLAST-CPU/README.md @@ -61,13 +61,12 @@ shipyard jobs add --jobs jobs-split.yaml --tail stdout.txt shipyard jobs add --jobs jobs-blast.yaml # poll the merge task until it completes -shipyard jobs tasks list --jobid blast --taskid merge-task-00001 --poll-until-tasks-complete +shipyard jobs tasks list --jobid blast --taskid merge-task-00000 --poll-until-tasks-complete # optionally egress the results.txt file from the compute node to local machine -shipyard data files task --filespec blast,merge-task-00001,wd/results.txt +shipyard data files task --filespec blast,merge-task-00000,wd/results.txt # clean up -shipyard jobs del -y --jobs jobs-split.yaml shipyard jobs del -y --jobs jobs-blast.yaml shipyard pool del -y ``` diff --git a/recipes/HPLinpack-Infiniband-IntelMPI/config/singularity/jobs.yaml b/recipes/HPLinpack-Infiniband-IntelMPI/config/singularity/jobs.yaml index d783fa0..9fc834f 100644 --- a/recipes/HPLinpack-Infiniband-IntelMPI/config/singularity/jobs.yaml +++ b/recipes/HPLinpack-Infiniband-IntelMPI/config/singularity/jobs.yaml @@ -18,6 +18,6 @@ job_specifications: mpi: runtime: intelmpi-ofa processes_per_node: 1 - pre_execution_command: source setup_hplinpack.sh -a $AVX -n 50000; source /opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh + pre_execution_command: source setup_hplinpack.sh -a $AVX -n 50000; module load mpi/impi; source $MPI_BIN/mpivars.sh # need to source mpivars again as the container intel mpi/benchmark is not the exact version as on the host command: /bin/bash -c "source /opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh; cd /opt/intel/mkl/benchmarks/mp_linpack && ./runme_intel64_prv -p $P -q $Q -b $B $PSIZE"