From 134262158be64ec6687ab7fbff09286ca72bcb79 Mon Sep 17 00:00:00 2001 From: Fred Park Date: Wed, 6 Nov 2019 04:14:40 +0000 Subject: [PATCH] Support Singularity image encryption - Modify singularity_images global resources to support encryption options - Automatically bind certificates to encrypted containers when a task executes --- cargo/task_file_mover.sh | 2 +- cascade/Dockerfile.singularity | 5 +- cascade/cascade.py | 39 +++++------ config_templates/config.yaml | 29 +++++--- convoy/fleet.py | 82 ++++++++++++++-------- convoy/settings.py | 123 ++++++++++++++++++++++----------- schemas/config.yaml | 39 +++++++++-- scripts/registry_login.sh | 4 +- scripts/shipyard_blobxfer.sh | 2 +- scripts/shipyard_cascade.sh | 0 scripts/shipyard_nodeprep.sh | 65 ++++++++++++++++- 11 files changed, 279 insertions(+), 111 deletions(-) mode change 100644 => 100755 scripts/shipyard_cascade.sh diff --git a/cargo/task_file_mover.sh b/cargo/task_file_mover.sh index f6c8de4..6103e99 100755 --- a/cargo/task_file_mover.sh +++ b/cargo/task_file_mover.sh @@ -4,7 +4,7 @@ set -e set -o pipefail set -f -privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/key.pem +privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/shipyard-enckey.pem for spec in "$@"; do IFS=',' read -ra parts <<< "$spec" diff --git a/cascade/Dockerfile.singularity b/cascade/Dockerfile.singularity index 5100a70..039c9c0 100644 --- a/cascade/Dockerfile.singularity +++ b/cascade/Dockerfile.singularity @@ -1,7 +1,7 @@ # Dockerfile for Azure/batch-shipyard (Cascade/Singularity) -# base image containing singularity -FROM alfpark/singularity:3.3.0 +# base image containing Singularity +FROM alfpark/singularity:3.4.2 FROM ubuntu:18.04 MAINTAINER Fred Park @@ -20,6 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libssl-dev \ libffi-dev \ squashfs-tools \ + cryptsetup-bin \ bash \ && python3 -m pip install --no-cache-dir --upgrade pip \ && pip3 install --no-cache-dir --upgrade setuptools wheel \ diff --git a/cascade/cascade.py b/cascade/cascade.py index 0594cb7..856d670 100755 --- a/cascade/cascade.py +++ b/cascade/cascade.py @@ -464,18 +464,19 @@ class ContainerImageSaveThread(threading.Thread): if username is not None and password is not None: credentials_command_argument = ( '--docker-username {} --docker-password {} '.format( - username, password)) + username, password) + ) else: credentials_command_argument = '' if image in _DIRECTDL_KEY_FINGERPRINT_DICT: singularity_pull_cmd = ( - 'singularity pull -F ' + - credentials_command_argument + - '{} {}'.format(image_out_path, image)) + 'singularity pull -F ' + credentials_command_argument + + '{} {}'.format(image_out_path, image) + ) key_fingerprint = _DIRECTDL_KEY_FINGERPRINT_DICT[image] if key_file_path.is_file(): - key_import_cmd = ('singularity key import {}' - .format(key_file_path)) + key_import_cmd = 'singularity key import {}'.format( + key_file_path) fingerprint_check_cmd = ( 'key_fingerprint=$({} | '.format(key_import_cmd) + 'grep -o "fingerprint \\(\\S*\\)" | ' + @@ -486,22 +487,19 @@ class ContainerImageSaveThread(threading.Thread): 'key file $key_fingerprint does not match ' + 'fingerprint provided {}")'.format(key_fingerprint) + ' && exit 1; fi') - cmd = (key_import_cmd + ' && ' + fingerprint_check_cmd + - ' && ' + singularity_pull_cmd) + cmd = '{} && {} && {}'.format( + key_import_cmd, fingerprint_check_cmd, + singularity_pull_cmd) else: - key_pull_cmd = ('singularity key pull {}' - .format(key_fingerprint)) - cmd = key_pull_cmd + ' && ' + singularity_pull_cmd - # if the image pulled from oras we need to manually - # verify the image - if image.startswith('oras://'): - singularity_verify_cmd = ('singularity verify {}' - .format(image_out_path)) - cmd = cmd + ' && ' + singularity_verify_cmd + cmd = '{} && singularity key pull {}'.format( + singularity_pull_cmd, key_fingerprint) + # always verify image separately + cmd = '{} && singularity verify {}'.format(cmd, image_out_path) else: - cmd = ('singularity pull -U -F ' + - credentials_command_argument + - '{} {}'.format(image_out_path, image)) + cmd = ( + 'singularity pull -U -F ' + credentials_command_argument + + '{} {}'.format(image_out_path, image) + ) return cmd def _pull(self, grtype: str, image: str) -> tuple: @@ -535,6 +533,7 @@ class ContainerImageSaveThread(threading.Thread): while True: rc, stdout, stderr = self._pull(grtype, image) if rc == 0: + logger.debug(stdout) break elif self._check_pull_output_overload(stderr.lower()): logger.error( diff --git a/config_templates/config.yaml b/config_templates/config.yaml index c188b31..f1dd354 100644 --- a/config_templates/config.yaml +++ b/config_templates/config.yaml @@ -27,16 +27,29 @@ global_resources: - myserver.azurecr.io/repo/myimage singularity_images: unsigned: - - shub://singularityhub/busybox - - shub://singularityhub/scientific-linux - - docker://busybox + - image: shub://singularityhub/busybox + - image: docker://busybox + - image: oras://myazurecr.azurecr.io/repo/myunsignedimage:1.0.0 + - image: library://user/repo/image:1.0.0 + - image: library://user/repo/encryptedimage:1.0.0 + encryption: + certificate: + sha1_thumbprint: 123456789... signed: - image: library://sylabs/tests/signed:1.0.0 - key_fingerprint: 8883491F4268F173C6E5DC49EDECE4F3F38D871E - key_file: /path/to/key/file - - image: oras://myazurecr.azurecr.io - key_fingerprint: 000123000123000123000123000123000123ABCD - key_file: /path/to/key/file + signing_key: + fingerprint: 8883491F4268F173C6E5DC49EDECE4F3F38D871E + - image: oras://myazurecr.azurecr.io/repo/mysignedimage:1.0.0 + signing_key: + fingerprint: 000123000123000123000123000123000123ABCD + file: /path/to/key/file + - image: library://user/repo/encryptedimage:1.0.0 + signing_key: + fingerprint: 000123000123000123000123000123000123ABCD + file: /path/to/key/file + encryption: + certificate: + sha1_thumbprint: 123456789... volumes: data_volumes: contdatavol: diff --git a/convoy/fleet.py b/convoy/fleet.py index a385eb0..6cdd2a2 100644 --- a/convoy/fleet.py +++ b/convoy/fleet.py @@ -1292,16 +1292,17 @@ def _construct_pool_object( 'credentials under batch') bi_pkg = _setup_batch_insights_package(config, pool_settings) _rflist.append((bi_pkg.name, bi_pkg)) - # singularity settings - singularity_signed_images_settings = ( - settings.global_resources_singularity_signed_images_settings(config)) - for image_settings in singularity_signed_images_settings: - if image_settings.key_file is None: - continue - key_file_name = util.singularity_image_name_to_key_file_name( - image_settings.image) - key_file_path = image_settings.key_file - _rflist.append((key_file_name, key_file_path)) + # singularity images settings + singularity_images_settings = ( + settings.global_resources_singularity_images_settings(config, False) + + settings.global_resources_singularity_images_settings(config, True) + ) + for image_settings in singularity_images_settings: + if util.is_not_empty(image_settings.key_file): + key_file_name = util.singularity_image_name_to_key_file_name( + image_settings.image) + key_file_path = image_settings.key_file + _rflist.append((key_file_name, key_file_path)) # get container registries docker_registries = settings.docker_registries(config) # set additional start task commands (pre version) @@ -1494,6 +1495,7 @@ def _construct_pool_object( task_scheduling_policy=task_scheduling_policy, certificate_references=[] ) + # certificate refs if encrypt: if is_windows: pool.certificate_references.append( @@ -1514,28 +1516,30 @@ def _construct_pool_object( visibility=[batchmodels.CertificateVisibility.start_task] ) ) + singularity_certs = [] + for image_setting in singularity_images_settings: + if util.is_not_empty( + image_setting.encryption_certificate_sha1_thumbprint): + pool.certificate_references.append( + batchmodels.CertificateReference( + thumbprint=image_setting. + encryption_certificate_sha1_thumbprint, + thumbprint_algorithm='sha1', + visibility=[batchmodels.CertificateVisibility.start_task] + ) + ) + singularity_certs.append( + image_setting.encryption_certificate_sha1_thumbprint) + del singularity_images_settings if util.is_not_empty(pool_settings.certificates): pool.certificate_references.extend(pool_settings.certificates) + # resource files for rf in sas_urls: pool.start_task.resource_files.append( batchmodels.ResourceFile( file_path=rf, http_url=sas_urls[rf]) ) - if not native or delay_image_preload: - pool.start_task.environment_settings.append( - batchmodels.EnvironmentSetting( - name='SHIPYARD_STORAGE_ENV', - value=crypto.encrypt_string( - encrypt, - '{}:{}:{}'.format( - storage.get_storageaccount(), - storage.get_storageaccount_endpoint(), - storage.get_storageaccount_key()), - config - ) - ) - ) if not native: if pool_settings.gpu_driver and util.is_none_or_empty(custom_image_na): pool.start_task.resource_files.append( @@ -1598,6 +1602,30 @@ def _construct_pool_object( endpoint_configuration=pec, public_ips=pool_settings.public_ips, ) + # storage env vars + if not native or delay_image_preload: + pool.start_task.environment_settings.append( + batchmodels.EnvironmentSetting( + name='SHIPYARD_STORAGE_ENV', + value=crypto.encrypt_string( + encrypt, + '{}:{}:{}'.format( + storage.get_storageaccount(), + storage.get_storageaccount_endpoint(), + storage.get_storageaccount_key()), + config + ) + ) + ) + # singularity certs + if util.is_not_empty(singularity_certs): + pool.start_task.environment_settings.append( + batchmodels.EnvironmentSetting( + name='SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES', + value=','.join(singularity_certs) + ) + ) + del singularity_certs # storage cluster settings if util.is_not_empty(sc_fstab_mounts): pool.start_task.environment_settings.append( @@ -1606,8 +1634,8 @@ def _construct_pool_object( value='#'.join(sc_fstab_mounts) ) ) - del sc_args - del sc_fstab_mounts + del sc_args + del sc_fstab_mounts # custom linux mount settings if util.is_not_empty(custom_linux_fstab_mounts): pool.start_task.environment_settings.append( @@ -1616,7 +1644,7 @@ def _construct_pool_object( value='#'.join(custom_linux_fstab_mounts) ) ) - del custom_linux_fstab_mounts + del custom_linux_fstab_mounts # add optional environment variables if not native and bs.store_timing_metrics: pool.start_task.environment_settings.append( diff --git a/convoy/settings.py b/convoy/settings.py index e465363..c5a75e5 100644 --- a/convoy/settings.py +++ b/convoy/settings.py @@ -512,9 +512,10 @@ SlurmCredentialsSettings = collections.namedtuple( 'db_password', ] ) -SingularitySignedImageSettings = collections.namedtuple( - 'SingularitySignedImageSettings', [ - 'image', 'key_fingerprint', 'key_file' +SingularityImageSettings = collections.namedtuple( + 'SingularityImageSettings', [ + 'image', 'key_fingerprint', 'key_file', + 'encryption_certificate_sha1_thumbprint' ] ) @@ -2398,62 +2399,74 @@ def global_resources_singularity_images(config): :rtype: list :return: all singularity images (signed and unsigned) """ - global_resources = _kv_read_checked(config, 'global_resources', default={}) - singularity_images = ( - _kv_read_checked(global_resources, 'singularity_images', default={})) - singularity_unsigned_images = ( - _kv_read_checked(singularity_images, 'unsigned', default=[])) + singularity_unsigned_images_settings = ( + global_resources_singularity_images_settings(config, False) + ) + singularity_unsigned_images = [ + settings.image for settings in singularity_unsigned_images_settings + ] singularity_signed_images_settings = ( - global_resources_singularity_signed_images_settings(config)) - singularity_signed_images = ( - [settings.image for settings in singularity_signed_images_settings]) + global_resources_singularity_images_settings(config, True) + ) + singularity_signed_images = [ + settings.image for settings in singularity_signed_images_settings + ] images = singularity_unsigned_images + singularity_signed_images singularity_signed_and_unsigned_images = ( set(singularity_unsigned_images).intersection( singularity_signed_images)) - if len(singularity_signed_and_unsigned_images): + if len(singularity_signed_and_unsigned_images) > 0: raise ValueError( - 'image(s) "{}" should not be both signed and unsigned' - .format('", "'.join(singularity_signed_and_unsigned_images))) + 'image(s) "{}" should not be both signed and unsigned'.format( + '", "'.join(singularity_signed_and_unsigned_images))) return images -def global_resources_singularity_signed_images_settings(config): - # type: (dict) -> list - """Get list of singularity signed images settings +def global_resources_singularity_images_settings(config, signed): + # type: (dict, bool) -> list + """Get list of singularity images settings :param dict config: configuration object + :param bool signed: get signed images if True, else unsigned images :rtype: list - :return: singularity signed images settings + :return: singularity images settings """ global_resources = _kv_read_checked(config, 'global_resources', default={}) - singularity_images = _kv_read_checked( + images = _kv_read_checked( global_resources, 'singularity_images', default={}) - singularity_signed_images = _kv_read_checked( - singularity_images, 'signed', default=[]) - singularity_signed_images_settings = [] - for settings in singularity_signed_images: + singularity_images = _kv_read_checked( + images, 'signed' if signed else 'unsigned', default=[]) + singularity_images_settings = [] + for settings in singularity_images: image = _kv_read_checked(settings, 'image') - if image is None: - raise ValueError('singularity signed image is invalid') - key_fingerprint = _kv_read_checked(settings, 'key_fingerprint') - if key_fingerprint is None: - raise ValueError('key_fingerprint for singularity signed image' - ' "{}" is invalid'.format(image)) - key_file = _kv_read_checked(settings, 'key_file') + if util.is_none_or_empty(image): + raise ValueError('singularity image is invalid') + key_fingerprint = None key_file_path = None - if key_file is not None: - key_file_path = pathlib.Path(key_file) - if not key_file_path.is_file(): - raise ValueError('invalid key file for image "{}"' - .format(image)) - singularity_signed_images_settings.append( - SingularitySignedImageSettings( + if signed: + key = _kv_read_checked(settings, 'signing_key', default={}) + key_fingerprint = _kv_read_checked(key, 'fingerprint') + if util.is_none_or_empty(key_fingerprint): + raise ValueError( + 'key_fingerprint for singularity signed image "{}" is ' + 'invalid'.format(image)) + key_file = _kv_read_checked(key, 'file') + if key_file is not None: + key_file_path = pathlib.Path(key_file) + if not key_file_path.is_file(): + raise ValueError( + 'invalid key file for image "{}"'.format(image)) + enc = _kv_read_checked(settings, 'encryption', default={}) + enc_cert = _kv_read_checked(enc, 'certificate', default={}) + singularity_images_settings.append( + SingularityImageSettings( image=image, key_fingerprint=key_fingerprint, key_file=key_file_path, + encryption_certificate_sha1_thumbprint=_kv_read_checked( + enc_cert, 'sha1_thumbprint'), ) ) - return singularity_signed_images_settings + return singularity_images_settings def singularity_signed_images_key_fingerprint_dict(config): @@ -2463,10 +2476,31 @@ def singularity_signed_images_key_fingerprint_dict(config): :rtype: dict :return: singularity signed images to key fingerprint """ + images_settings = global_resources_singularity_images_settings( + config, True) + return dict( + (settings.image, settings.key_fingerprint) + for settings in images_settings + ) + + +def singularity_image_to_encryption_cert_map(config): + # type: (dict) -> dict + """Get mapping of image to encryptiong cert thumbprint + :param dict config: configuration object + :rtype: dict + :return: singularity image name to cert thumbprint + """ images_settings = ( - global_resources_singularity_signed_images_settings(config)) - return dict((settings.image, settings.key_fingerprint) - for settings in images_settings) + global_resources_singularity_images_settings(config, False) + + global_resources_singularity_images_settings(config, True) + ) + image_map = {} + for image in images_settings: + if util.is_not_empty(image.encryption_certificate_sha1_thumbprint): + image_map[ + image.image] = image.encryption_certificate_sha1_thumbprint + return image_map def global_resources_files(config): @@ -4041,6 +4075,13 @@ def task_settings( if username is not None and password is not None: env_vars['SINGULARITY_DOCKER_USERNAME'] = username env_vars['SINGULARITY_DOCKER_PASSWORD'] = password + singularity_cert_map = singularity_image_to_encryption_cert_map(config) + cert = singularity_cert_map.get(singularity_image) + if cert is not None: + # use run option over env var to use az batch env var to cert path + run_opts.append( + '--pem-path=$AZ_BATCH_NODE_STARTUP_DIR/certs/' + 'sha1-{}-rsa.pem'.format(cert)) # constraints max_task_retries = _kv_read(conf, 'max_task_retries') max_wall_time = _kv_read_checked(conf, 'max_wall_time') diff --git a/schemas/config.yaml b/schemas/config.yaml index 48606b8..bc17b06 100644 --- a/schemas/config.yaml +++ b/schemas/config.yaml @@ -77,7 +77,21 @@ mapping: unsigned: type: seq sequence: - - type: str + - type: map + mapping: + image: + type: str + required: true + encryption: + type: map + mapping: + certificate: + type: map + required: true + mapping: + sha1_thumbprint: + type: str + required: true signed: type: seq sequence: @@ -86,11 +100,24 @@ mapping: image: type: str required: true - key_fingerprint: - type: str - required: true - key_file: - type: str + signing_key: + type: map + mapping: + fingerprint: + type: str + required: true + file: + type: str + encryption: + type: map + mapping: + certificate: + type: map + required: true + mapping: + sha1_thumbprint: + type: str + required: true volumes: type: map mapping: diff --git a/scripts/registry_login.sh b/scripts/registry_login.sh index a0dc785..55e71fe 100755 --- a/scripts/registry_login.sh +++ b/scripts/registry_login.sh @@ -53,10 +53,10 @@ docker_login() { # decrypt passwords if necessary if [ "$1" == "-e" ]; then if [ -n "$DOCKER_LOGIN_PASSWORD" ]; then - DOCKER_LOGIN_PASSWORD=$(echo "$DOCKER_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/key.pem) + DOCKER_LOGIN_PASSWORD=$(echo "$DOCKER_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/shipyard-enckey.pem) fi if [ -n "$SINGULARITY_LOGIN_PASSWORD" ]; then - SINGULARITY_LOGIN_PASSWORD=$(echo "$SINGULARITY_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/key.pem) + SINGULARITY_LOGIN_PASSWORD=$(echo "$SINGULARITY_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey ../certs/shipyard-enckey.pem) fi fi diff --git a/scripts/shipyard_blobxfer.sh b/scripts/shipyard_blobxfer.sh index ddcb7d7..499d77e 100755 --- a/scripts/shipyard_blobxfer.sh +++ b/scripts/shipyard_blobxfer.sh @@ -18,7 +18,7 @@ for spec in "$@"; do eo=${parts[5]} cond=${parts[6]} # decrypt ciphertext - privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/key.pem + privatekey=$AZ_BATCH_NODE_STARTUP_DIR/certs/shipyard-enckey.pem cipher=$(echo "$cipher" | base64 -d | openssl rsautl -decrypt -inkey "$privatekey") IFS=',' read -ra storage <<< "$cipher" sa=${storage[0]} diff --git a/scripts/shipyard_cascade.sh b/scripts/shipyard_cascade.sh old mode 100644 new mode 100755 diff --git a/scripts/shipyard_nodeprep.sh b/scripts/shipyard_nodeprep.sh index 13847b6..01e91f1 100755 --- a/scripts/shipyard_nodeprep.sh +++ b/scripts/shipyard_nodeprep.sh @@ -187,7 +187,9 @@ while getopts "h?abcde:fg:i:jkl:m:no:p:qrs:tuv:wx:yz:" opt; do lis=$OPTARG ;; m) + OLD_IFS=$IFS IFS=',' read -ra sc_args <<< "${OPTARG,,}" + IFS=$OLD_IFS ;; n) native_mode=1 @@ -726,7 +728,9 @@ install_nvidia_software() { # check for nvidia card check_for_nvidia_card # split arg into two + local OLD_IFS=$IFS IFS=':' read -ra GPUARGS <<< "$gpu" + IFS=$OLD_IFS local is_viz=${GPUARGS[0]} local nvdriver=${GPUARGS[1]} # remove nouveau @@ -1090,7 +1094,9 @@ install_kata_containers() { process_fstab_entry() { local desc=$1 local fstab_entry=$2 + local OLD_IFS=$IFS IFS=' ' read -ra fs <<< "$fstab_entry" + IFS=$OLD_IFS local mountpoint="${fs[1]}" log INFO "Creating host directory for $desc at $mountpoint" mkdir -p "$mountpoint" @@ -1126,6 +1132,7 @@ mount_storage_clusters() { if [ -n "$SHIPYARD_STORAGE_CLUSTER_FSTAB" ]; then log DEBUG "Mounting storage clusters" local fstab_mounts + local OLD_IFS=$IFS IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_STORAGE_CLUSTER_FSTAB" for fstab in "${fstab_mounts[@]}"; do # eval and split fstab var to expand vars @@ -1133,6 +1140,7 @@ mount_storage_clusters() { IFS=' ' read -ra parts <<< "$fstab_entry" mount "${parts[1]}" done + IFS=$OLD_IFS log INFO "Storage clusters mounted" fi } @@ -1142,6 +1150,7 @@ process_storage_clusters() { log DEBUG "Processing storage clusters" # eval and split fstab var to expand vars (this is ok since it is set by shipyard) local fstab_mounts + local OLD_IFS=$IFS fstab_mounts=$(eval echo "$SHIPYARD_STORAGE_CLUSTER_FSTAB") IFS='#' read -ra fstabs <<< "$fstab_mounts" i=0 @@ -1151,6 +1160,7 @@ process_storage_clusters() { process_fstab_entry "$sc_arg" "$fstab_entry" i=$((i + 1)) done + IFS=$OLD_IFS log INFO "Storage clusters processed" fi } @@ -1159,6 +1169,7 @@ mount_custom_fstab() { if [ -n "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" ]; then log DEBUG "Mounting custom mounts via fstab" local fstab_mounts + local OLD_IFS=$IFS IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" for fstab in "${fstab_mounts[@]}"; do # eval and split fstab var to expand vars @@ -1166,6 +1177,7 @@ mount_custom_fstab() { IFS=' ' read -ra parts <<< "$fstab_entry" mount "${parts[1]}" done + IFS=$OLD_IFS log INFO "Custom mounts via fstab mounted" fi } @@ -1174,6 +1186,7 @@ process_custom_fstab() { if [ -n "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" ]; then log DEBUG "Processing custom mounts via fstab" local fstab_mounts + local OLD_IFS=$IFS IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" for fstab in "${fstab_mounts[@]}"; do # eval and split fstab var to expand vars @@ -1181,17 +1194,49 @@ process_custom_fstab() { IFS=' ' read -ra parts <<< "$fstab_entry" process_fstab_entry "${parts[2]}" "$fstab_entry" done + IFS=$OLD_IFS log INFO "Custom mounts via fstab processed" fi } +convert_singularity_certificates() { + # converts pfx certs to PKCS1 (RSA private) pem certs for singularity + if [ -z "$SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES" ]; then + log INFO "No singularity decryption certificates defined" + return + fi + log INFO "Processing Singularity decryption certificates: $SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES" + local OLD_IFS=$IFS + IFS=',' read -ra cert_tps <<< "${SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES,,}" + IFS=$OLD_IFS + local pfx + local pw + local pkcs1 + pushd "$AZ_BATCH_CERTIFICATES_DIR" + for tp in "${cert_tps[@]}"; do + pfx="sha1-${tp}.pfx" + pw="${pfx}.pw" + if [ ! -f "$pfx" ] || [ ! -f "$pw" ]; then + log ERROR "Certificate needed for Singularity image decryption not found: $pfx or $pw" + exit 1 + fi + pkcs1="sha1-${tp}-rsa.pem" + openssl pkcs12 -in "$pfx" -nodes -password "file:${pw}" | openssl rsa -out "$pkcs1" + chmod 640 "$pkcs1" + chown _azbatch:_azbatchsudogrp "$pkcs1" + # remove the pfx/pw files bound to start task + rm -f "$pfx" "$pw" + done + popd +} + decrypt_encrypted_credentials() { # convert pfx to pem pfxfile=$AZ_BATCH_CERTIFICATES_DIR/sha1-$encrypted.pfx - privatekey=$AZ_BATCH_CERTIFICATES_DIR/key.pem - openssl pkcs12 -in "$pfxfile" -out "$privatekey" -nodes -password file:"${pfxfile}".pw + privatekey=$AZ_BATCH_CERTIFICATES_DIR/shipyard-enckey.pem + openssl pkcs12 -in "$pfxfile" -out "$privatekey" -nodes -password "file:${pfxfile}.pw" # remove pfx-related files - rm -f "$pfxfile" "${pfxfile}".pw + rm -f "$pfxfile" "${pfxfile}.pw" # decrypt creds SHIPYARD_STORAGE_ENV=$(echo "$SHIPYARD_STORAGE_ENV" | base64 -d | openssl rsautl -decrypt -inkey "$privatekey") if [[ -n ${DOCKER_LOGIN_USERNAME+x} ]]; then @@ -1372,7 +1417,9 @@ check_for_storage_cluster_software() { fi local rc for sc_arg in "${sc_args[@]}"; do + local OLD_IFS=$IFS IFS=':' read -ra sc <<< "$sc_arg" + IFS=$OLD_IFS local server_type=${sc[0]} if [ "$server_type" == "nfs" ]; then set +e @@ -1407,7 +1454,9 @@ install_storage_cluster_dependencies() { local repo="http://download.opensuse.org/repositories/filesystems/${repodir}/filesystems.repo" fi for sc_arg in "${sc_args[@]}"; do + local OLD_IFS=$IFS IFS=':' read -ra sc <<< "$sc_arg" + IFS=$OLD_IFS server_type=${sc[0]} if [ "$server_type" == "nfs" ]; then if [ "$PACKAGER" == "apt" ]; then @@ -1677,6 +1726,7 @@ install_and_start_node_exporter() { else ib="--no-collector.infiniband" fi + local OLD_IFS=$IFS if [ -n "${sc_args[0]}" ]; then for sc_arg in "${sc_args[@]}"; do IFS=':' read -ra sc <<< "$sc_arg" @@ -1685,9 +1735,12 @@ install_and_start_node_exporter() { break fi done + IFS=$OLD_IFS fi local pneo + OLD_IFS=$IFS IFS=',' read -ra pneo <<< "$PROM_NODE_EXPORTER_OPTIONS" + IFS=$OLD_IFS # shellcheck disable=SC2086 "${AZ_BATCH_TASK_WORKING_DIR}"/node_exporter \ "$ib" $nfs \ @@ -1715,7 +1768,9 @@ install_and_start_cadvisor() { # start local pcao if [ -n "${PROM_CADVISOR_OPTIONS}" ]; then + local OLD_IFS=$IFS IFS=',' read -ra pcao <<< "$PROM_CADVISOR_OPTIONS" + IFS=$OLD_IFS else pcao=() fi @@ -1769,6 +1824,7 @@ echo "Docker image preload delay: $delay_preload" echo "Cascade via container: $cascadecontainer" echo "Concurrent source downloads: $concurrent_source_downloads" echo "Block on images: $block" +echo "Singularity decryption certs: $SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES" echo "" # set python env vars @@ -1804,6 +1860,9 @@ if [ -n "$encrypted" ]; then decrypt_encrypted_credentials fi +# convert certs to pkcs1 for singularity encrypted container support +convert_singularity_certificates + # create shared mount points mkdir -p "$MOUNTS_PATH"