From 69235ba2c1ce555300bb949a057c685e84f20027 Mon Sep 17 00:00:00 2001 From: Fred Park Date: Wed, 25 Apr 2018 08:19:27 -0700 Subject: [PATCH] Add default_working_dir option - Clarify where a container runs by default in the jobs config doc - Resolves #190 --- config_templates/jobs.yaml | 2 ++ convoy/settings.py | 28 +++++++++++++------- docs/14-batch-shipyard-configuration-jobs.md | 26 ++++++++++++++++++ schemas/jobs.yaml | 6 +++++ 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/config_templates/jobs.yaml b/config_templates/jobs.yaml index 636ad1f..d8d9a3f 100644 --- a/config_templates/jobs.yaml +++ b/config_templates/jobs.yaml @@ -59,6 +59,7 @@ job_specifications: include: - jobdata*.bin blobxfer_extra_options: null + default_working_dir: batch tasks: - id: null docker_image: busybox @@ -190,6 +191,7 @@ job_specifications: include: - 'out*.dat' blobxfer_extra_options: null + default_working_dir: batch remove_container_after_exit: true shm_size: 256m additional_docker_run_options: [] diff --git a/convoy/settings.py b/convoy/settings.py index 5915567..d04a409 100644 --- a/convoy/settings.py +++ b/convoy/settings.py @@ -3181,6 +3181,16 @@ def task_settings(cloud_pool, config, poolconf, jobspec, conf): del tdv # binding order matters for Singularity bindparm = '-v' if util.is_not_empty(docker_image) else '-B' + # get working dir default + def_wd = _kv_read_checked( + conf, 'default_working_dir', + default=_kv_read_checked(jobspec, 'default_working_dir') + ) + if util.is_none_or_empty(def_wd) or def_wd == 'batch': + if is_windows: + def_wd = '%AZ_BATCH_TASK_WORKING_DIR%' + else: + def_wd = '$AZ_BATCH_TASK_WORKING_DIR' # bind root dir and set working dir if not native: # mount batch root dir @@ -3193,16 +3203,14 @@ def task_settings(cloud_pool, config, poolconf, jobspec, conf): '{} $AZ_BATCH_NODE_ROOT_DIR:$AZ_BATCH_NODE_ROOT_DIR'.format( bindparm)) # set working directory if not already set - if util.is_not_empty(docker_image): - if not any((x.startswith('-w ') or x.startswith('--workdir ')) - for x in run_opts): - if is_windows: - run_opts.append('-w %AZ_BATCH_TASK_WORKING_DIR%') - else: - run_opts.append('-w $AZ_BATCH_TASK_WORKING_DIR') - else: - if not any(x.startswith('--pwd ') for x in run_opts): - run_opts.append('--pwd $AZ_BATCH_TASK_WORKING_DIR') + if def_wd != 'container': + if util.is_not_empty(docker_image): + if not any((x.startswith('-w ') or x.startswith('--workdir ')) + for x in run_opts): + run_opts.append('-w {}'.format(def_wd)) + else: + if not any(x.startswith('--pwd ') for x in run_opts): + run_opts.append('--pwd {}'.format(def_wd)) if util.is_not_empty(data_volumes): dv = global_resources_data_volumes(config) for dvkey in data_volumes: diff --git a/docs/14-batch-shipyard-configuration-jobs.md b/docs/14-batch-shipyard-configuration-jobs.md index ae4f197..245dc46 100644 --- a/docs/14-batch-shipyard-configuration-jobs.md +++ b/docs/14-batch-shipyard-configuration-jobs.md @@ -2,6 +2,19 @@ This page contains in-depth details on how to configure the jobs configuration file for Batch Shipyard. +## Considerations +- By default, containers are executed with the working directory set to the +Azure Batch task working directory (or `$AZ_BATCH_TASK_WORKING_DIR`). This +default is set such that all files written to this path (or any child paths) +are persisted to the host. This behavior can be changed through the options +`default_working_dir` or to explicitly set the working dir with the +container runtime respective switch to `additional_docker_run_options` +or `additional_singularity_options`. +- Container logs are automatically captured to `stdout.txt` and `stderr.txt` +within the `$AZ_BATCH_TASK_DIR`. You can egress these logs using the +`data files task` command (e.g., +`data files task --filespec myjobid,mytaskid,stdout.txt`). + ## Schema The jobs schema is as follows: @@ -67,6 +80,7 @@ job_specifications: include: - jobdata*.bin blobxfer_extra_options: null + default_working_dir: batch tasks: - id: null docker_image: busybox @@ -196,6 +210,7 @@ job_specifications: include: - 'out*.dat' blobxfer_extra_options: null + default_working_dir: batch remove_container_after_exit: true shm_size: 256m additional_docker_run_options: [] @@ -456,6 +471,17 @@ transferred again. This object currently supports `azure_batch` and * (optional) `exclude` property defines optional exclude filters. * (optional) `blobxfer_extra_options` are any extra options to pass to `blobxfer`. +* (optional) `default_working_dir` defines the default working directory +when the container executes. Valid values for this property are `batch` and +`container`. If `batch` is selected, the working directory for the container +execution is set to the Azure Batch task working directory, or +`$AZ_BATCH_TASK_WORKING_DIR`. If `container` is selected, the working +directory for the container execution is not explicitly set. The default is +`batch` if not specified. If you wish to specify a different working +directory, you can pass the appropriate working directory parameter to the +container runtime through either `additional_docker_run_options` or +`additional_singularity_options`. A working directory option specified within +that property takes precedence over this option. The required `tasks` property is an array of tasks to add to the job: diff --git a/schemas/jobs.yaml b/schemas/jobs.yaml index 1f671fb..c03b72e 100644 --- a/schemas/jobs.yaml +++ b/schemas/jobs.yaml @@ -152,6 +152,9 @@ mapping: - type: str blobxfer_extra_options: type: str + default_working_dir: + type: str + enum: ['batch', 'container'] tasks: type: seq sequence: @@ -471,6 +474,9 @@ mapping: - type: str blobxfer_extra_options: type: str + default_working_dir: + type: str + enum: ['batch', 'container'] remove_container_after_exit: type: bool shm_size: