Support Singularity image encryption

- Modify singularity_images global resources to support encryption
options
- Automatically bind certificates to encrypted containers when a task
executes

cargo/task_file_mover.sh

@@ -4,7 +4,7 @@ set -e
 set -o pipefail
 set -f
 
-privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/key.pem
+privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/shipyard-enckey.pem
 
 for spec in "$@"; do
     IFS=',' read -ra parts <<< "$spec"

cascade/Dockerfile.singularity

@@ -1,7 +1,7 @@
 # Dockerfile for Azure/batch-shipyard (Cascade/Singularity)
 
-# base image containing singularity
+# base image containing Singularity
-FROM alfpark/singularity:3.3.0
+FROM alfpark/singularity:3.4.2
 
 FROM ubuntu:18.04
 MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
@@ -20,6 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libssl-dev \
         libffi-dev \
         squashfs-tools \
+        cryptsetup-bin \
         bash \
     && python3 -m pip install --no-cache-dir --upgrade pip \
     && pip3 install --no-cache-dir --upgrade setuptools wheel \

cascade/cascade.py

@@ -464,18 +464,19 @@ class ContainerImageSaveThread(threading.Thread):
         if username is not None and password is not None:
             credentials_command_argument = (
                 '--docker-username {} --docker-password {} '.format(
-                    username, password))
+                    username, password)
+            )
         else:
             credentials_command_argument = ''
         if image in _DIRECTDL_KEY_FINGERPRINT_DICT:
             singularity_pull_cmd = (
-                'singularity pull -F ' +
+                'singularity pull -F ' + credentials_command_argument +
-                credentials_command_argument +
+                '{} {}'.format(image_out_path, image)
-                '{} {}'.format(image_out_path, image))
+            )
             key_fingerprint = _DIRECTDL_KEY_FINGERPRINT_DICT[image]
             if key_file_path.is_file():
-                key_import_cmd = ('singularity key import {}'
+                key_import_cmd = 'singularity key import {}'.format(
-                                  .format(key_file_path))
+                    key_file_path)
                 fingerprint_check_cmd = (
                     'key_fingerprint=$({} | '.format(key_import_cmd) +
                     'grep -o "fingerprint \\(\\S*\\)" | ' +
@@ -486,22 +487,19 @@ class ContainerImageSaveThread(threading.Thread):
                     'key file $key_fingerprint does not match ' +
                     'fingerprint provided {}")'.format(key_fingerprint) +
                     ' && exit 1; fi')
-                cmd = (key_import_cmd + ' && ' + fingerprint_check_cmd +
+                cmd = '{} && {} && {}'.format(
-                       ' && ' + singularity_pull_cmd)
+                    key_import_cmd, fingerprint_check_cmd,
+                    singularity_pull_cmd)
             else:
-                key_pull_cmd = ('singularity key pull {}'
+                cmd = '{} && singularity key pull {}'.format(
-                                .format(key_fingerprint))
+                    singularity_pull_cmd, key_fingerprint)
-                cmd = key_pull_cmd + ' && ' + singularity_pull_cmd
+            # always verify image separately
-            # if the image pulled from oras we need to manually
+            cmd = '{} && singularity verify {}'.format(cmd, image_out_path)
-            # verify the image
-            if image.startswith('oras://'):
-                singularity_verify_cmd = ('singularity verify {}'
-                                          .format(image_out_path))
-                cmd = cmd + ' && ' + singularity_verify_cmd
         else:
-            cmd = ('singularity pull -U -F ' +
+            cmd = (
-                   credentials_command_argument +
+                'singularity pull -U -F ' + credentials_command_argument +
-                   '{} {}'.format(image_out_path, image))
+                '{} {}'.format(image_out_path, image)
+            )
         return cmd
 
     def _pull(self, grtype: str, image: str) -> tuple:
@@ -535,6 +533,7 @@ class ContainerImageSaveThread(threading.Thread):
         while True:
             rc, stdout, stderr = self._pull(grtype, image)
             if rc == 0:
+                logger.debug(stdout)
                 break
             elif self._check_pull_output_overload(stderr.lower()):
                 logger.error(

config_templates/config.yaml

@@ -27,16 +27,29 @@ global_resources:
   - myserver.azurecr.io/repo/myimage
   singularity_images:
     unsigned:
-    - shub://singularityhub/busybox
+    - image: shub://singularityhub/busybox
-    - shub://singularityhub/scientific-linux
+    - image: docker://busybox
-    - docker://busybox
+    - image: oras://myazurecr.azurecr.io/repo/myunsignedimage:1.0.0
+    - image: library://user/repo/image:1.0.0
+    - image: library://user/repo/encryptedimage:1.0.0
+      encryption:
+        certificate:
+          sha1_thumbprint: 123456789...
     signed:
     - image: library://sylabs/tests/signed:1.0.0
-      key_fingerprint: 8883491F4268F173C6E5DC49EDECE4F3F38D871E
+      signing_key:
-      key_file: /path/to/key/file
+        fingerprint: 8883491F4268F173C6E5DC49EDECE4F3F38D871E
-    - image: oras://myazurecr.azurecr.io
+    - image: oras://myazurecr.azurecr.io/repo/mysignedimage:1.0.0
-      key_fingerprint: 000123000123000123000123000123000123ABCD
+      signing_key:
-      key_file: /path/to/key/file
+        fingerprint: 000123000123000123000123000123000123ABCD
+        file: /path/to/key/file
+    - image: library://user/repo/encryptedimage:1.0.0
+      signing_key:
+        fingerprint: 000123000123000123000123000123000123ABCD
+        file: /path/to/key/file
+      encryption:
+        certificate:
+          sha1_thumbprint: 123456789...
   volumes:
     data_volumes:
       contdatavol:

convoy/fleet.py

@@ -1292,16 +1292,17 @@ def _construct_pool_object(
                 'credentials under batch')
         bi_pkg = _setup_batch_insights_package(config, pool_settings)
         _rflist.append((bi_pkg.name, bi_pkg))
-    # singularity settings
+    # singularity images settings
-    singularity_signed_images_settings = (
+    singularity_images_settings = (
-        settings.global_resources_singularity_signed_images_settings(config))
+        settings.global_resources_singularity_images_settings(config, False) +
-    for image_settings in singularity_signed_images_settings:
+        settings.global_resources_singularity_images_settings(config, True)
-        if image_settings.key_file is None:
+    )
-            continue
+    for image_settings in singularity_images_settings:
-        key_file_name = util.singularity_image_name_to_key_file_name(
+        if util.is_not_empty(image_settings.key_file):
-            image_settings.image)
+            key_file_name = util.singularity_image_name_to_key_file_name(
-        key_file_path = image_settings.key_file
+                image_settings.image)
-        _rflist.append((key_file_name, key_file_path))
+            key_file_path = image_settings.key_file
+            _rflist.append((key_file_name, key_file_path))
     # get container registries
     docker_registries = settings.docker_registries(config)
     # set additional start task commands (pre version)
@@ -1494,6 +1495,7 @@ def _construct_pool_object(
         task_scheduling_policy=task_scheduling_policy,
         certificate_references=[]
     )
+    # certificate refs
     if encrypt:
         if is_windows:
             pool.certificate_references.append(
@@ -1514,28 +1516,30 @@ def _construct_pool_object(
                     visibility=[batchmodels.CertificateVisibility.start_task]
                 )
             )
+    singularity_certs = []
+    for image_setting in singularity_images_settings:
+        if util.is_not_empty(
+                image_setting.encryption_certificate_sha1_thumbprint):
+            pool.certificate_references.append(
+                batchmodels.CertificateReference(
+                    thumbprint=image_setting.
+                    encryption_certificate_sha1_thumbprint,
+                    thumbprint_algorithm='sha1',
+                    visibility=[batchmodels.CertificateVisibility.start_task]
+                )
+            )
+            singularity_certs.append(
+                image_setting.encryption_certificate_sha1_thumbprint)
+    del singularity_images_settings
     if util.is_not_empty(pool_settings.certificates):
         pool.certificate_references.extend(pool_settings.certificates)
+    # resource files
     for rf in sas_urls:
         pool.start_task.resource_files.append(
             batchmodels.ResourceFile(
                 file_path=rf,
                 http_url=sas_urls[rf])
         )
-    if not native or delay_image_preload:
-        pool.start_task.environment_settings.append(
-            batchmodels.EnvironmentSetting(
-                name='SHIPYARD_STORAGE_ENV',
-                value=crypto.encrypt_string(
-                    encrypt,
-                    '{}:{}:{}'.format(
-                        storage.get_storageaccount(),
-                        storage.get_storageaccount_endpoint(),
-                        storage.get_storageaccount_key()),
-                    config
-                )
-            )
-        )
     if not native:
         if pool_settings.gpu_driver and util.is_none_or_empty(custom_image_na):
             pool.start_task.resource_files.append(
@@ -1598,6 +1602,30 @@ def _construct_pool_object(
             endpoint_configuration=pec,
             public_ips=pool_settings.public_ips,
         )
+    # storage env vars
+    if not native or delay_image_preload:
+        pool.start_task.environment_settings.append(
+            batchmodels.EnvironmentSetting(
+                name='SHIPYARD_STORAGE_ENV',
+                value=crypto.encrypt_string(
+                    encrypt,
+                    '{}:{}:{}'.format(
+                        storage.get_storageaccount(),
+                        storage.get_storageaccount_endpoint(),
+                        storage.get_storageaccount_key()),
+                    config
+                )
+            )
+        )
+    # singularity certs
+    if util.is_not_empty(singularity_certs):
+        pool.start_task.environment_settings.append(
+            batchmodels.EnvironmentSetting(
+                name='SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES',
+                value=','.join(singularity_certs)
+            )
+        )
+    del singularity_certs
     # storage cluster settings
     if util.is_not_empty(sc_fstab_mounts):
         pool.start_task.environment_settings.append(
@@ -1606,8 +1634,8 @@ def _construct_pool_object(
                 value='#'.join(sc_fstab_mounts)
             )
         )
-        del sc_args
+    del sc_args
-        del sc_fstab_mounts
+    del sc_fstab_mounts
     # custom linux mount settings
     if util.is_not_empty(custom_linux_fstab_mounts):
         pool.start_task.environment_settings.append(
@@ -1616,7 +1644,7 @@ def _construct_pool_object(
                 value='#'.join(custom_linux_fstab_mounts)
             )
         )
-        del custom_linux_fstab_mounts
+    del custom_linux_fstab_mounts
     # add optional environment variables
     if not native and bs.store_timing_metrics:
         pool.start_task.environment_settings.append(

convoy/settings.py

@@ -512,9 +512,10 @@ SlurmCredentialsSettings = collections.namedtuple(
         'db_password',
     ]
 )
-SingularitySignedImageSettings = collections.namedtuple(
+SingularityImageSettings = collections.namedtuple(
-    'SingularitySignedImageSettings', [
+    'SingularityImageSettings', [
-        'image', 'key_fingerprint', 'key_file'
+        'image', 'key_fingerprint', 'key_file',
+        'encryption_certificate_sha1_thumbprint'
     ]
 )
 
@@ -2398,62 +2399,74 @@ def global_resources_singularity_images(config):
     :rtype: list
     :return: all singularity images (signed and unsigned)
     """
-    global_resources = _kv_read_checked(config, 'global_resources', default={})
+    singularity_unsigned_images_settings = (
-    singularity_images = (
+        global_resources_singularity_images_settings(config, False)
-        _kv_read_checked(global_resources, 'singularity_images', default={}))
+    )
-    singularity_unsigned_images = (
+    singularity_unsigned_images = [
-        _kv_read_checked(singularity_images, 'unsigned', default=[]))
+        settings.image for settings in singularity_unsigned_images_settings
+    ]
     singularity_signed_images_settings = (
-        global_resources_singularity_signed_images_settings(config))
+        global_resources_singularity_images_settings(config, True)
-    singularity_signed_images = (
+    )
-        [settings.image for settings in singularity_signed_images_settings])
+    singularity_signed_images = [
+        settings.image for settings in singularity_signed_images_settings
+    ]
     images = singularity_unsigned_images + singularity_signed_images
     singularity_signed_and_unsigned_images = (
         set(singularity_unsigned_images).intersection(
             singularity_signed_images))
-    if len(singularity_signed_and_unsigned_images):
+    if len(singularity_signed_and_unsigned_images) > 0:
         raise ValueError(
-            'image(s) "{}" should not be both signed and unsigned'
+            'image(s) "{}" should not be both signed and unsigned'.format(
-            .format('", "'.join(singularity_signed_and_unsigned_images)))
+                '", "'.join(singularity_signed_and_unsigned_images)))
     return images
 
 
-def global_resources_singularity_signed_images_settings(config):
+def global_resources_singularity_images_settings(config, signed):
-    # type: (dict) -> list
+    # type: (dict, bool) -> list
-    """Get list of singularity signed images settings
+    """Get list of singularity images settings
     :param dict config: configuration object
+    :param bool signed: get signed images if True, else unsigned images
     :rtype: list
-    :return: singularity signed images settings
+    :return: singularity images settings
     """
     global_resources = _kv_read_checked(config, 'global_resources', default={})
-    singularity_images = _kv_read_checked(
+    images = _kv_read_checked(
         global_resources, 'singularity_images', default={})
-    singularity_signed_images = _kv_read_checked(
+    singularity_images = _kv_read_checked(
-        singularity_images, 'signed', default=[])
+        images, 'signed' if signed else 'unsigned', default=[])
-    singularity_signed_images_settings = []
+    singularity_images_settings = []
-    for settings in singularity_signed_images:
+    for settings in singularity_images:
         image = _kv_read_checked(settings, 'image')
-        if image is None:
+        if util.is_none_or_empty(image):
-            raise ValueError('singularity signed image is invalid')
+            raise ValueError('singularity image is invalid')
-        key_fingerprint = _kv_read_checked(settings, 'key_fingerprint')
+        key_fingerprint = None
-        if key_fingerprint is None:
-            raise ValueError('key_fingerprint for singularity signed image'
-                             ' "{}" is invalid'.format(image))
-        key_file = _kv_read_checked(settings, 'key_file')
         key_file_path = None
-        if key_file is not None:
+        if signed:
-            key_file_path = pathlib.Path(key_file)
+            key = _kv_read_checked(settings, 'signing_key', default={})
-            if not key_file_path.is_file():
+            key_fingerprint = _kv_read_checked(key, 'fingerprint')
-                raise ValueError('invalid key file for image "{}"'
+            if util.is_none_or_empty(key_fingerprint):
-                                 .format(image))
+                raise ValueError(
-        singularity_signed_images_settings.append(
+                    'key_fingerprint for singularity signed image "{}" is '
-            SingularitySignedImageSettings(
+                    'invalid'.format(image))
+            key_file = _kv_read_checked(key, 'file')
+            if key_file is not None:
+                key_file_path = pathlib.Path(key_file)
+                if not key_file_path.is_file():
+                    raise ValueError(
+                        'invalid key file for image "{}"'.format(image))
+        enc = _kv_read_checked(settings, 'encryption', default={})
+        enc_cert = _kv_read_checked(enc, 'certificate', default={})
+        singularity_images_settings.append(
+            SingularityImageSettings(
                 image=image,
                 key_fingerprint=key_fingerprint,
                 key_file=key_file_path,
+                encryption_certificate_sha1_thumbprint=_kv_read_checked(
+                    enc_cert, 'sha1_thumbprint'),
             )
         )
-    return singularity_signed_images_settings
+    return singularity_images_settings
 
 
 def singularity_signed_images_key_fingerprint_dict(config):
@@ -2463,10 +2476,31 @@ def singularity_signed_images_key_fingerprint_dict(config):
     :rtype: dict
     :return: singularity signed images to key fingerprint
     """
+    images_settings = global_resources_singularity_images_settings(
+        config, True)
+    return dict(
+        (settings.image, settings.key_fingerprint)
+        for settings in images_settings
+    )
+
+
+def singularity_image_to_encryption_cert_map(config):
+    # type: (dict) -> dict
+    """Get mapping of image to encryptiong cert thumbprint
+    :param dict config: configuration object
+    :rtype: dict
+    :return: singularity image name to cert thumbprint
+    """
     images_settings = (
-        global_resources_singularity_signed_images_settings(config))
+        global_resources_singularity_images_settings(config, False) +
-    return dict((settings.image, settings.key_fingerprint)
+        global_resources_singularity_images_settings(config, True)
-                for settings in images_settings)
+    )
+    image_map = {}
+    for image in images_settings:
+        if util.is_not_empty(image.encryption_certificate_sha1_thumbprint):
+            image_map[
+                image.image] = image.encryption_certificate_sha1_thumbprint
+    return image_map
 
 
 def global_resources_files(config):
@@ -4041,6 +4075,13 @@ def task_settings(
             if username is not None and password is not None:
                 env_vars['SINGULARITY_DOCKER_USERNAME'] = username
                 env_vars['SINGULARITY_DOCKER_PASSWORD'] = password
+        singularity_cert_map = singularity_image_to_encryption_cert_map(config)
+        cert = singularity_cert_map.get(singularity_image)
+        if cert is not None:
+            # use run option over env var to use az batch env var to cert path
+            run_opts.append(
+                '--pem-path=$AZ_BATCH_NODE_STARTUP_DIR/certs/'
+                'sha1-{}-rsa.pem'.format(cert))
     # constraints
     max_task_retries = _kv_read(conf, 'max_task_retries')
     max_wall_time = _kv_read_checked(conf, 'max_wall_time')

schemas/config.yaml

@@ -77,7 +77,21 @@ mapping:
           unsigned:
             type: seq
             sequence:
-              - type: str
+              - type: map
+                mapping:
+                  image:
+                    type: str
+                    required: true
+                  encryption:
+                    type: map
+                    mapping:
+                      certificate:
+                        type: map
+                        required: true
+                        mapping:
+                          sha1_thumbprint:
+                            type: str
+                            required: true
           signed:
             type: seq
             sequence:
@@ -86,11 +100,24 @@ mapping:
                   image:
                     type: str
                     required: true
-                  key_fingerprint:
+                  signing_key:
-                    type: str
+                    type: map
-                    required: true
+                    mapping:
-                  key_file:
+                      fingerprint:
-                    type: str
+                        type: str
+                        required: true
+                      file:
+                        type: str
+                  encryption:
+                    type: map
+                    mapping:
+                      certificate:
+                        type: map
+                        required: true
+                        mapping:
+                          sha1_thumbprint:
+                            type: str
+                            required: true
       volumes:
         type: map
         mapping:

scripts/registry_login.sh

@@ -53,10 +53,10 @@ docker_login() {
 # decrypt passwords if necessary
 if [ "$1" == "-e" ]; then
     if [ -n "$DOCKER_LOGIN_PASSWORD" ]; then
-        DOCKER_LOGIN_PASSWORD=$(echo "$DOCKER_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/key.pem)
+        DOCKER_LOGIN_PASSWORD=$(echo "$DOCKER_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/shipyard-enckey.pem)
     fi
     if [ -n "$SINGULARITY_LOGIN_PASSWORD" ]; then
-        SINGULARITY_LOGIN_PASSWORD=$(echo "$SINGULARITY_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/key.pem)
+        SINGULARITY_LOGIN_PASSWORD=$(echo "$SINGULARITY_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/shipyard-enckey.pem)
     fi
 fi
 

scripts/shipyard_blobxfer.sh

@@ -18,7 +18,7 @@ for spec in "$@"; do
         eo=${parts[5]}
         cond=${parts[6]}
         # decrypt ciphertext
-        privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/key.pem
+        privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/shipyard-enckey.pem
         cipher=$(echo "$cipher" | base64 -d | openssl rsautl -decrypt -inkey "$privatekey")
         IFS=',' read -ra storage <<< "$cipher"
         sa=${storage[0]}

scripts/shipyard_nodeprep.sh

@@ -187,7 +187,9 @@ while getopts "h?abcde:fg:i:jkl:m:no:p:qrs:tuv:wx:yz:" opt; do
             lis=$OPTARG
             ;;
         m)
+            OLD_IFS=$IFS
             IFS=',' read -ra sc_args <<< "${OPTARG,,}"
+            IFS=$OLD_IFS
             ;;
         n)
             native_mode=1
@@ -726,7 +728,9 @@ install_nvidia_software() {
     # check for nvidia card
     check_for_nvidia_card
     # split arg into two
+    local OLD_IFS=$IFS
     IFS=':' read -ra GPUARGS <<< "$gpu"
+    IFS=$OLD_IFS
     local is_viz=${GPUARGS[0]}
     local nvdriver=${GPUARGS[1]}
     # remove nouveau
@@ -1090,7 +1094,9 @@ install_kata_containers() {
 process_fstab_entry() {
     local desc=$1
     local fstab_entry=$2
+    local OLD_IFS=$IFS
     IFS=' ' read -ra fs <<< "$fstab_entry"
+    IFS=$OLD_IFS
     local mountpoint="${fs[1]}"
     log INFO "Creating host directory for $desc at $mountpoint"
     mkdir -p "$mountpoint"
@@ -1126,6 +1132,7 @@ mount_storage_clusters() {
     if [ -n "$SHIPYARD_STORAGE_CLUSTER_FSTAB" ]; then
         log DEBUG "Mounting storage clusters"
         local fstab_mounts
+        local OLD_IFS=$IFS
         IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_STORAGE_CLUSTER_FSTAB"
         for fstab in "${fstab_mounts[@]}"; do
             # eval and split fstab var to expand vars
@@ -1133,6 +1140,7 @@ mount_storage_clusters() {
             IFS=' ' read -ra parts <<< "$fstab_entry"
             mount "${parts[1]}"
         done
+        IFS=$OLD_IFS
         log INFO "Storage clusters mounted"
     fi
 }
@@ -1142,6 +1150,7 @@ process_storage_clusters() {
         log DEBUG "Processing storage clusters"
         # eval and split fstab var to expand vars (this is ok since it is set by shipyard)
         local fstab_mounts
+        local OLD_IFS=$IFS
         fstab_mounts=$(eval echo "$SHIPYARD_STORAGE_CLUSTER_FSTAB")
         IFS='#' read -ra fstabs <<< "$fstab_mounts"
         i=0
@@ -1151,6 +1160,7 @@ process_storage_clusters() {
             process_fstab_entry "$sc_arg" "$fstab_entry"
             i=$((i + 1))
         done
+        IFS=$OLD_IFS
         log INFO "Storage clusters processed"
     fi
 }
@@ -1159,6 +1169,7 @@ mount_custom_fstab() {
     if [ -n "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" ]; then
         log DEBUG "Mounting custom mounts via fstab"
         local fstab_mounts
+        local OLD_IFS=$IFS
         IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_CUSTOM_MOUNTS_FSTAB"
         for fstab in "${fstab_mounts[@]}"; do
             # eval and split fstab var to expand vars
@@ -1166,6 +1177,7 @@ mount_custom_fstab() {
             IFS=' ' read -ra parts <<< "$fstab_entry"
             mount "${parts[1]}"
         done
+        IFS=$OLD_IFS
         log INFO "Custom mounts via fstab mounted"
     fi
 }
@@ -1174,6 +1186,7 @@ process_custom_fstab() {
     if [ -n "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" ]; then
         log DEBUG "Processing custom mounts via fstab"
         local fstab_mounts
+        local OLD_IFS=$IFS
         IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_CUSTOM_MOUNTS_FSTAB"
         for fstab in "${fstab_mounts[@]}"; do
             # eval and split fstab var to expand vars
@@ -1181,17 +1194,49 @@ process_custom_fstab() {
             IFS=' ' read -ra parts <<< "$fstab_entry"
             process_fstab_entry "${parts[2]}" "$fstab_entry"
         done
+        IFS=$OLD_IFS
         log INFO "Custom mounts via fstab processed"
     fi
 }
 
+convert_singularity_certificates() {
+    # converts pfx certs to PKCS1 (RSA private) pem certs for singularity
+    if [ -z "$SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES" ]; then
+        log INFO "No singularity decryption certificates defined"
+        return
+    fi
+    log INFO "Processing Singularity decryption certificates: $SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES"
+    local OLD_IFS=$IFS
+    IFS=',' read -ra cert_tps <<< "${SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES,,}"
+    IFS=$OLD_IFS
+    local pfx
+    local pw
+    local pkcs1
+    pushd "$AZ_BATCH_CERTIFICATES_DIR"
+    for tp in "${cert_tps[@]}"; do
+        pfx="sha1-${tp}.pfx"
+        pw="${pfx}.pw"
+        if [ ! -f "$pfx" ] || [ ! -f "$pw" ]; then
+            log ERROR "Certificate needed for Singularity image decryption not found: $pfx or $pw"
+            exit 1
+        fi
+        pkcs1="sha1-${tp}-rsa.pem"
+        openssl pkcs12 -in "$pfx" -nodes -password "file:${pw}" | openssl rsa -out "$pkcs1"
+        chmod 640 "$pkcs1"
+        chown _azbatch:_azbatchsudogrp "$pkcs1"
+        # remove the pfx/pw files bound to start task
+        rm -f "$pfx" "$pw"
+    done
+    popd
+}
+
 decrypt_encrypted_credentials() {
     # convert pfx to pem
     pfxfile=$AZ_BATCH_CERTIFICATES_DIR/sha1-$encrypted.pfx
-    privatekey=$AZ_BATCH_CERTIFICATES_DIR/key.pem
+    privatekey=$AZ_BATCH_CERTIFICATES_DIR/shipyard-enckey.pem
-    openssl pkcs12 -in "$pfxfile" -out "$privatekey" -nodes -password file:"${pfxfile}".pw
+    openssl pkcs12 -in "$pfxfile" -out "$privatekey" -nodes -password "file:${pfxfile}.pw"
     # remove pfx-related files
-    rm -f "$pfxfile" "${pfxfile}".pw
+    rm -f "$pfxfile" "${pfxfile}.pw"
     # decrypt creds
     SHIPYARD_STORAGE_ENV=$(echo "$SHIPYARD_STORAGE_ENV" | base64 -d | openssl rsautl -decrypt -inkey "$privatekey")
     if [[ -n ${DOCKER_LOGIN_USERNAME+x} ]]; then
@@ -1372,7 +1417,9 @@ check_for_storage_cluster_software() {
     fi
     local rc
     for sc_arg in "${sc_args[@]}"; do
+        local OLD_IFS=$IFS
         IFS=':' read -ra sc <<< "$sc_arg"
+        IFS=$OLD_IFS
         local server_type=${sc[0]}
         if [ "$server_type" == "nfs" ]; then
             set +e
@@ -1407,7 +1454,9 @@ install_storage_cluster_dependencies() {
         local repo="http://download.opensuse.org/repositories/filesystems/${repodir}/filesystems.repo"
     fi
     for sc_arg in "${sc_args[@]}"; do
+        local OLD_IFS=$IFS
         IFS=':' read -ra sc <<< "$sc_arg"
+        IFS=$OLD_IFS
         server_type=${sc[0]}
         if [ "$server_type" == "nfs" ]; then
             if [ "$PACKAGER" == "apt" ]; then
@@ -1677,6 +1726,7 @@ install_and_start_node_exporter() {
     else
         ib="--no-collector.infiniband"
     fi
+    local OLD_IFS=$IFS
     if [ -n "${sc_args[0]}" ]; then
         for sc_arg in "${sc_args[@]}"; do
             IFS=':' read -ra sc <<< "$sc_arg"
@@ -1685,9 +1735,12 @@ install_and_start_node_exporter() {
                 break
             fi
         done
+        IFS=$OLD_IFS
     fi
     local pneo
+    OLD_IFS=$IFS
     IFS=',' read -ra pneo <<< "$PROM_NODE_EXPORTER_OPTIONS"
+    IFS=$OLD_IFS
     # shellcheck disable=SC2086
     "${AZ_BATCH_TASK_WORKING_DIR}"/node_exporter \
         "$ib" $nfs \
@@ -1715,7 +1768,9 @@ install_and_start_cadvisor() {
     # start
     local pcao
     if [ -n "${PROM_CADVISOR_OPTIONS}" ]; then
+        local OLD_IFS=$IFS
         IFS=',' read -ra pcao <<< "$PROM_CADVISOR_OPTIONS"
+        IFS=$OLD_IFS
     else
         pcao=()
     fi
@@ -1769,6 +1824,7 @@ echo "Docker image preload delay: $delay_preload"
 echo "Cascade via container: $cascadecontainer"
 echo "Concurrent source downloads: $concurrent_source_downloads"
 echo "Block on images: $block"
+echo "Singularity decryption certs: $SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES"
 echo ""
 
 # set python env vars
@@ -1804,6 +1860,9 @@ if [ -n "$encrypted" ]; then
     decrypt_encrypted_credentials
 fi
 
+# convert certs to pkcs1 for singularity encrypted container support
+convert_singularity_certificates
+
 # create shared mount points
 mkdir -p "$MOUNTS_PATH"