From d7d5faaf7a9391ac05c8fe8797088092f59c9d35 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 9 Feb 2018 09:56:53 -0800 Subject: [PATCH] Fix: Custom scripts not read from cluster.yaml (#388) --- .style.yapf | 6 ++ .vscode/launch.json | 67 +++++++++++++++++++ .vscode/settings.json | 5 +- cli/config.py | 31 +++++++-- cli/spark/endpoints/cluster/cluster_create.py | 28 +------- requirements.txt | 8 ++- setup.cfg | 2 - 7 files changed, 107 insertions(+), 40 deletions(-) create mode 100644 .style.yapf create mode 100644 .vscode/launch.json delete mode 100644 setup.cfg diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 00000000..d48c5e3e --- /dev/null +++ b/.style.yapf @@ -0,0 +1,6 @@ +[style] +based_on_style = pep8 +spaces_before_comment = 4 +split_before_logical_operator = true +indent_width = 4 +column_limit = 120 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..442bbeed --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,67 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "List clusters", + "type": "python", + "request": "launch", + "stopOnEntry": false, + "internalConsoleOptions": "openOnSessionStart", + "pythonPath": "${config:python.pythonPath}", + "program": "${workspaceFolder}/cli/entrypoint.py", + "cwd": "${workspaceFolder}", + "args": [ + "spark", "cluster", "list" + ], + "env": {}, + "envFile": "${workspaceFolder}/.env", + "debugOptions": [ + "RedirectOutput" + ] + }, + { + "name": "Create cluster", + "type": "python", + "request": "launch", + "stopOnEntry": false, + "internalConsoleOptions": "openOnSessionStart", + "pythonPath": "${config:python.pythonPath}", + "program": "${workspaceFolder}/cli/entrypoint.py", + "cwd": "${workspaceFolder}", + "args": [ + "spark", "cluster", "create", "--id", "spark-debug" + ], + "env": {}, + "envFile": "${workspaceFolder}/.env", + "debugOptions": [ + "RedirectOutput" + ] + }, + { + "name": "Python: Attach", + "type": "python", + "request": "attach", + "localRoot": "${workspaceFolder}", + "remoteRoot": "${workspaceFolder}", + "port": 3000, + "secret": "my_secret", + "host": "localhost" + }, + { + "name": "Python: Terminal (integrated)", + "type": "python", + "request": "launch", + "stopOnEntry": true, + "pythonPath": "${config:python.pythonPath}", + "program": "${file}", + "cwd": "", + "console": "integratedTerminal", + "env": {}, + "envFile": "${workspaceFolder}/.env", + "debugOptions": [] + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index d2747727..08c910c0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,6 +8,7 @@ "**/__pycache__/**/*": true }, "python.autoComplete.extraPaths": [ - "${workspaceRoot}/node" - ] + "${workspaceRoot}/node_scripts" + ], + "python.formatting.provider": "yapf" } diff --git a/cli/config.py b/cli/config.py index d302bba5..20184c7c 100644 --- a/cli/config.py +++ b/cli/config.py @@ -101,9 +101,9 @@ def _merge_secrets_dict(secrets: SecretsConfiguration, secrets_config): docker_config = secrets_config.get('docker') if docker_config: secrets.docker = DockerConfiguration( - endpoint = docker_config.get('endpoint'), - username = docker_config.get('username'), - password = docker_config.get('password'), + endpoint=docker_config.get('endpoint'), + username=docker_config.get('username'), + password=docker_config.get('password'), ) default_config = secrets_config.get('default') @@ -132,6 +132,7 @@ def read_cluster_config(path: str = aztk.utils.constants.DEFAULT_CLUSTER_CONFIG_ return cluster_config_from_dict(config_dict) + def cluster_config_from_dict(config: dict): output = ClusterConfiguration() wait = False @@ -161,10 +162,26 @@ def cluster_config_from_dict(config: dict): output.user_configuration.password = config['password'] if config.get('custom_scripts') not in [[None], None]: - output.custom_scripts = config['custom_scripts'] + output.custom_scripts = [] + for custom_script in config['custom_scripts']: + output.custom_scripts.append( + aztk.spark.models.CustomScript( + script=custom_script['script'], + run_on=custom_script['runOn'] + ) + ) if config.get('azure_files') not in [[None], None]: - output.file_shares = config['azure_files'] + output.file_shares = [] + for file_share in config['azure_files']: + output.file_shares.append( + aztk.spark.models.FileShare( + storage_account_name=file_share['storage_account_name'], + storage_account_key=file_share['storage_account_key'], + file_share_path=file_share['file_share_path'], + mount_path=file_share['mount_path'], + ) + ) if config.get('docker_repo') is not None: output.docker_repo = config['docker_repo'] @@ -175,7 +192,6 @@ def cluster_config_from_dict(config: dict): return output, wait - class SshConfig: def __init__(self): @@ -271,13 +287,14 @@ class SshConfig: raise aztk.error.AztkError( "Please supply a username either in the ssh.yaml configuration file or with a parameter (--username)") + class JobConfig(): def __init__(self): self.id = None self.applications = [] self.custom_scripts = None self.spark_configuration = None - self.vm_size=None + self.vm_size = None self.docker_repo = None self.max_dedicated_nodes = None self.max_low_pri_nodes = None diff --git a/cli/spark/endpoints/cluster/cluster_create.py b/cli/spark/endpoints/cluster/cluster_create.py index 4a910c6d..1279e7f9 100644 --- a/cli/spark/endpoints/cluster/cluster_create.py +++ b/cli/spark/endpoints/cluster/cluster_create.py @@ -54,32 +54,6 @@ def execute(args: typing.NamedTuple): docker_repo=args.docker_repo)) wait = wait if args.wait is None else args.wait - if cluster_conf.custom_scripts: - custom_scripts = [] - for custom_script in cluster_conf.custom_scripts: - custom_scripts.append( - aztk.spark.models.CustomScript( - script=custom_script['script'], - run_on=custom_script['runOn'] - ) - ) - else: - custom_scripts = None - - if cluster_conf.file_shares: - file_shares = [] - for file_share in cluster_conf.file_shares: - file_shares.append( - aztk.spark.models.FileShare( - storage_account_name=file_share['storage_account_name'], - storage_account_key=file_share['storage_account_key'], - file_share_path=file_share['file_share_path'], - mount_path=file_share['mount_path'] - ) - ) - else: - file_shares = None - user_configuration = cluster_conf.user_configuration if user_configuration and user_configuration.username: @@ -123,7 +97,7 @@ def print_cluster_conf(cluster_conf: ClusterConfiguration, wait: bool): log.info("> dedicated: %s", cluster_conf.vm_count) log.info("> low priority: %s", cluster_conf.vm_low_pri_count) log.info("spark cluster vm size: %s", cluster_conf.vm_size) - log.info("custom scripts: %s", cluster_conf.custom_scripts) + log.info("custom scripts: %s", len(cluster_conf.custom_scripts) if cluster_conf.custom_scripts else 0) log.info("subnet ID: %s", cluster_conf.subnet_id) log.info("file shares: %s", len(cluster_conf.file_shares) if cluster_conf.file_shares is not None else 0) log.info("docker repo name: %s", cluster_conf.docker_repo) diff --git a/requirements.txt b/requirements.txt index 364c2b52..e59410e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,13 @@ +# Distribution azure-batch==3.0.0 azure-mgmt-batch==5.0.0 azure-mgmt-storage==1.5.0 azure-storage==0.33.0 -pytest==3.1.3 -pylint==1.7.2 pyyaml==3.12 pycryptodome==3.4.7 paramiko==2.4.0 + +# Development +yapf==0.20.1 +pylint==1.7.2 +pytest==3.1.3 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 4879cb6d..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[pep8] -max-line-length = 160