Azure
/
aztk
зеркало из https://github.com/Azure/aztk.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Enable docker authentication for private images (#92) 

* Docker login working

* fix docker image ignored

* Fix job need to wait for pool to be created

* Fix

* fix

* Fix

* Fix

* Update docs

* Fix cr

* Fix merge issue
This commit is contained in:
Timothee Guerin 2017-10-03 13:30:12 -07:00 коммит произвёл GitHub
Родитель 952d8c85a9
Коммит f51f613fe7
6 изменённых файлов: 152 добавлений и 102 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

37
aztk/clusterlib.py
Просмотреть файл

@ -30,6 +30,8 @@ def cluster_install_cmd(
to be run on the start task of the pool to setup spark.
"""
docker_repo = docker_repo or constants.DEFAULT_DOCKER_REPO
ret = [
'apt-get -y clean',
'apt-get -y update',
@ -40,7 +42,7 @@ def cluster_install_cmd(
'chmod 777 $AZ_BATCH_TASK_WORKING_DIR/setup_node.sh',
'/bin/bash $AZ_BATCH_TASK_WORKING_DIR/setup_node.sh {0} {1} "{2}"'.format(
constants.DOCKER_SPARK_CONTAINER_NAME,
constants.DEFAULT_DOCKER_REPO,
docker_repo,
docker_run_cmd(docker_repo)),
]
@ -52,9 +54,6 @@ def docker_run_cmd(docker_repo: str = None) -> str:
Build the docker run command by setting up the environment variables
"""
docker_repo = docker_repo if docker_repo != None \
else constants.DEFAULT_DOCKER_REPO
cmd = CommandBuilder('docker run')
cmd.add_option('--net', 'host')
cmd.add_option('--name', constants.DOCKER_SPARK_CONTAINER_NAME)
@ -104,6 +103,7 @@ def generate_cluster_start_task(
# TODO use certificate
batch_config = azure_api.get_batch_config()
blob_config = azure_api.get_blob_config()
environment_settings = [
batch_models.EnvironmentSetting(
name="BATCH_ACCOUNT_KEY", value=batch_config.account_key),
@ -123,7 +123,7 @@ def generate_cluster_start_task(
name="SPARK_JUPYTER_PORT", value=spark_jupyter_port),
batch_models.EnvironmentSetting(
name="SPARK_WEB_UI_PORT", value=spark_web_ui_port),
]
] + get_docker_credentials()
# start task command
command = cluster_install_cmd(zip_resource_file, docker_repo)
@ -174,6 +174,25 @@ def clean_up_custom_scripts():
if os.path.exists(os.path.join(constants.ROOT_PATH, constants.CUSTOM_SCRIPTS_DEST)):
rmtree(constants.CUSTOM_SCRIPTS_DEST)
def get_docker_credentials():
creds = []
secrets_config = config.SecretsConfig()
secrets_config.load_secrets_config()
if secrets_config.docker_endpoint:
creds.append(batch_models.EnvironmentSetting(
name="DOCKER_ENDPOINT", value=secrets_config.docker_endpoint))
if secrets_config.docker_username:
creds.append(batch_models.EnvironmentSetting(
name="DOCKER_USERNAME", value=secrets_config.docker_username))
if secrets_config.docker_password:
creds.append(batch_models.EnvironmentSetting(
name="DOCKER_PASSWORD", value=secrets_config.docker_password))
return creds
def create_cluster(
custom_scripts: List[object],
cluster_id: str,
@ -251,9 +270,7 @@ def create_cluster(
"Password is empty, cannot add user to cluster. Provide a ssh public key in .aztk/secrets.yaml. Or provide an ssh-key or password with commnad line parameters (--ssh-key or --password).")
# Create the pool + create user for the pool
util.create_pool_if_not_exist(
pool,
wait)
util.create_pool_if_not_exist(pool)
# Create job
job = batch_models.JobAddParameter(
@ -332,7 +349,7 @@ def pretty_node_count(cluster: Cluster) -> str:
def pretty_dedicated_node_count(cluster: Cluster)-> str:
if (cluster.pool.allocation_state is batch_models.AllocationState.resizing\
if (cluster.pool.allocation_state is batch_models.AllocationState.resizing
or cluster.pool.state is batch_models.PoolState.deleting)\
and cluster.dedicated_nodes != cluster.target_dedicated_nodes:
return '{} -> {}'.format(
@ -343,7 +360,7 @@ def pretty_dedicated_node_count(cluster: Cluster)-> str:
def pretty_low_pri_node_count(cluster: Cluster)-> str:
if (cluster.pool.allocation_state is batch_models.AllocationState.resizing\
if (cluster.pool.allocation_state is batch_models.AllocationState.resizing
or cluster.pool.state is batch_models.PoolState.deleting)\
and cluster.low_pri_nodes != cluster.target_low_pri_nodes:
return '{} -> {}'.format(

84
aztk/config.py
Просмотреть файл

@ -6,6 +6,7 @@ from aztk import log
from aztk import error
from . import constants
def load_spark_config():
"""
Copies the spark-defautls.conf and spark-env.sh in the .aztk/ diretory
@ -15,7 +16,8 @@ def load_spark_config():
try:
copyfile(
os.path.join(constants.DEFAULT_SPARK_CONF_SOURCE, 'spark-defaults.conf'),
os.path.join(constants.DEFAULT_SPARK_CONF_SOURCE,
'spark-defaults.conf'),
os.path.join(constants.DEFAULT_SPARK_CONF_DEST, 'spark-defaults.conf'))
log.info("Loaded spark-defaults.conf")
except Exception as e:
@ -49,10 +51,12 @@ class SecretsConfig:
self.storage_account_key = None
self.storage_account_suffix = None
self.docker_endpoint = None
self.docker_username = None
self.docker_password = None
self.ssh_pub_key = None
self.ssh_priv_key = None
def load_secrets_config(self, path: str=constants.DEFAULT_SECRETS_PATH):
"""
Loads the secrets.yaml file in the .aztk directory
@ -70,44 +74,52 @@ class SecretsConfig:
self._merge_dict(secrets_config)
def _merge_dict(self, secrets_config):
# Ensure all necessary fields are provided
try:
self.batch_account_name = secrets_config['batch']['batchaccountname']
except KeyError:
raise error.AztkError("Please specify a batch account name in your .aztk/secrets.yaml file")
raise error.AztkError(
"Please specify a batch account name in your .aztk/secrets.yaml file")
try:
self.batch_account_key = secrets_config['batch']['batchaccountkey']
except KeyError:
raise error.AztkError("Please specify a batch account key in your .aztk/secrets.yaml file")
raise error.AztkError(
"Please specify a batch account key in your .aztk/secrets.yaml file")
try:
self.batch_service_url = secrets_config['batch']['batchserviceurl']
except KeyError:
raise error.AztkError("Please specify a batch service in your .aztk/secrets.yaml file")
raise error.AztkError(
"Please specify a batch service in your .aztk/secrets.yaml file")
try:
self.storage_account_name = secrets_config['storage']['storageaccountname']
except KeyError:
raise error.AztkError("Please specify a storage account name in your .aztk/secrets.yaml file")
raise error.AztkError(
"Please specify a storage account name in your .aztk/secrets.yaml file")
try:
self.storage_account_key = secrets_config['storage']['storageaccountkey']
except KeyError:
raise error.AztkError("Please specify a storage account key in your .aztk/secrets.yaml file")
raise error.AztkError(
"Please specify a storage account key in your .aztk/secrets.yaml file")
try:
self.storage_account_suffix = secrets_config['storage']['storageaccountsuffix']
except KeyError:
raise error.AztkError("Please specify a storage account suffix in your .aztk/secrets.yaml file")
raise error.AztkError(
"Please specify a storage account suffix in your .aztk/secrets.yaml file")
docker_config = secrets_config.get('docker')
if docker_config:
self.docker_endpoint = docker_config.get('endpoint')
self.docker_username = docker_config.get('username')
self.docker_password = docker_config.get('password')
default_config = secrets_config.get('default')
# Check for ssh keys if they are provided
try:
self.ssh_priv_key = secrets_config['default']['ssh_priv_key']
except (KeyError, TypeError) as e:
pass
try:
self.ssh_pub_key = secrets_config['default']['ssh_pub_key']
except (KeyError, TypeError) as e:
pass
if default_config:
self.ssh_priv_key = default_config.get('ssh_priv_key')
self.ssh_pub_key = default_config.get('ssh_pub_key')
class ClusterConfig:
@ -124,7 +136,6 @@ class ClusterConfig:
self.docker_repo = None
self.wait = None
def _read_config_file(self, path: str=constants.DEFAULT_CLUSTER_CONFIG_PATH):
"""
Reads the config file in the .aztk/ directory (.aztk/cluster.yaml)
@ -145,7 +156,6 @@ class ClusterConfig:
self._merge_dict(config)
def _merge_dict(self, config):
if 'id' in config and config['id'] is not None:
self.uid = config['id']
@ -174,7 +184,6 @@ class ClusterConfig:
if 'wait' in config and config['wait'] is not None:
self.wait = config['wait']
def merge(self, uid, username, size, size_low_pri, vm_size, ssh_key, password, wait, docker_repo):
"""
Reads configuration file (cluster.yaml), merges with command line parameters,
@ -184,16 +193,16 @@ class ClusterConfig:
self._merge_dict(
dict(
id = uid,
username = username,
size = size,
size_low_pri = size_low_pri,
vm_size = vm_size,
ssh_key = ssh_key,
password = password,
wait = wait,
custom_scripts = None,
docker_repo = docker_repo
id=uid,
username=username,
size=size,
size_low_pri=size_low_pri,
vm_size=vm_size,
ssh_key=ssh_key,
password=password,
wait=wait,
custom_scripts=None,
docker_repo=docker_repo
)
)
@ -228,7 +237,6 @@ class SshConfig:
self.jupyter_port = None
self.connect = True
def _read_config_file(self, path: str=constants.DEFAULT_SSH_CONFIG_PATH):
"""
Reads the config file in the .aztk/ directory (.aztk/cluster.yaml)
@ -249,7 +257,6 @@ class SshConfig:
self._merge_dict(config)
def _merge_dict(self, config):
if 'username' in config and config['username'] is not None:
self.username = config['username']
@ -269,7 +276,6 @@ class SshConfig:
if 'connect' in config and config['connect'] is False:
self.connect = False
def merge(self, cluster_id, username, job_ui_port, web_ui_port, jupyter_port, connect):
"""
Merges fields with args object
@ -277,12 +283,12 @@ class SshConfig:
self._read_config_file()
self._merge_dict(
dict(
cluster_id = cluster_id,
username = username,
job_ui_port = job_ui_port,
web_ui_port = web_ui_port,
jupyter_port = jupyter_port,
connect = connect
cluster_id=cluster_id,
username=username,
job_ui_port=job_ui_port,
web_ui_port=web_ui_port,
jupyter_port=jupyter_port,
connect=connect
)
)

12
aztk/util.py
Просмотреть файл

@ -47,7 +47,7 @@ class MasterInvalidStateError(Exception):
def wait_for_master_to_be_ready(cluster_id: str):
batch_client = azure_api.get_batch_client()
master_node_id = None
log.info("Waiting for spark master to be ready")
start_time = datetime.datetime.now()
while True:
if not master_node_id:
@ -153,7 +153,7 @@ def get_master_node_id(pool_id):
return get_master_node_id_from_pool(batch_client.pool.get(pool_id))
def create_pool_if_not_exist(pool, wait=True):
def create_pool_if_not_exist(pool):
"""
Creates the specified pool if it doesn't already exist
:param batch_client: The batch client to use.
@ -166,14 +166,6 @@ def create_pool_if_not_exist(pool, wait=True):
try:
batch_client.pool.add(pool)
if wait:
wait_for_all_nodes_state(pool, frozenset(
(batch_models.ComputeNodeState.start_task_failed,
batch_models.ComputeNodeState.unusable,
batch_models.ComputeNodeState.idle)
))
return True
else:
return True
except batch_models.BatchErrorException as e:
if e.error.code != "PoolExists":

6
config/secrets.yaml.template
Просмотреть файл

@ -10,6 +10,12 @@ storage:
storageaccountkey:
storageaccountsuffix: core.windows.net
# Configuration for private docker repositories. If using public containers you do not need to provide authentification
docker:
# username:
# password:
# endpoint:
default:
# SSH keys used to create a user and connect to a server.
# The public key can either be the public key itself (ssh-rsa ...) or the path to the ssh key.

17
docs/12-docker-image.md
Просмотреть файл

@ -37,4 +37,19 @@ Once you have your Docker image built and hosted publicly, you can then use the
## Using a custom Docker Image that is Privately Hosted
_Coming soon_
To use a private docker image you will need to provide a docker username and password that have access to the repository you want to use.
In `.thunderbolt/secrets.yaml` setup your docker config
```yaml
docker:
username: <myusername>
password: <mypassword>
```
If your private repository is not on docker hub (Azure container registry for example) you can provide the endpoint here too
```yaml
docker:
username: <myusername>
password: <mypassword>
endpoint: <https://my-custom-docker-endpoint.com>
```

16
node_scripts/setup_node.sh
Просмотреть файл

@ -20,7 +20,21 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
apt-get -y update
apt-get -y install docker-ce
docker pull container_name
if [ -z "$DOCKER_USERNAME" ]; then
echo "No Credentials provided. No need to login to dockerhub $DOCKER_USERNAME $DOCKER_PASSWORD"
else
echo "Docker credentials provided. Login in."
docker login $docker_endpoint --username $DOCKER_USERNAME --password $DOCKER_PASSWORD
fi
if [ -z "$DOCKER_ENDPOINT" ]; then
echo "Pulling $repo_name from dockerhub"
docker pull $repo_name
else
echo "Pulling $container_name from $DOCKER_ENDPOINT"
docker pull $DOCKER_ENDPOINT/$repo_name
fi
# Unzip resource files and set permissions
apt-get -y install unzip