зеркало из https://github.com/Azure/aztk.git
103 строки
2.5 KiB
Python
Executable File
103 строки
2.5 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
from redbull import sparklib
|
|
|
|
try:
|
|
import configparser
|
|
except ImportError:
|
|
import ConfigParser as configparser
|
|
|
|
import os
|
|
import datetime
|
|
import random
|
|
import argparse
|
|
|
|
import azure.batch.batch_service_client as batch
|
|
import azure.batch.batch_auth as batch_auth
|
|
import azure.batch.models as batch_models
|
|
import azure.storage.blob as blob
|
|
|
|
# config file path
|
|
_config_path = os.path.join(os.path.dirname(__file__), '../configuration.cfg')
|
|
|
|
if __name__ == '__main__':
|
|
|
|
_pool_id = None
|
|
_app_id = None
|
|
_username = 'admin'
|
|
_password = 'pass123!'
|
|
|
|
# parse arguments
|
|
parser = argparse.ArgumentParser(prog="az_spark")
|
|
|
|
parser.add_argument("--cluster-id", required=True,
|
|
help="the unique name of your spark cluster")
|
|
parser.add_argument("--app-id", required=True,
|
|
help="the unique name of your spark app")
|
|
parser.add_argument("-u", "--user",
|
|
help="the relative path to your spark app in your directory")
|
|
parser.add_argument("-p", "--password",
|
|
help="the relative path to your spark app in your directory")
|
|
|
|
args = parser.parse_args()
|
|
|
|
print()
|
|
if args.cluster_id is not None:
|
|
_pool_id = args.cluster_id
|
|
print("spark cluster id: %s" % _pool_id)
|
|
|
|
if args.app_id is not None:
|
|
_app_id = args.app_id
|
|
print("spark job id: %s" % _app_id)
|
|
|
|
if args.user is not None:
|
|
_username = args.user
|
|
print("az_spark username: %s" % _username)
|
|
|
|
if args.password is not None:
|
|
_password = args.password
|
|
print("az_spark password: %s" % _password)
|
|
|
|
# Read config file
|
|
global_config = configparser.ConfigParser()
|
|
global_config.read(_config_path)
|
|
|
|
# Set up the configuration
|
|
batch_account_key = global_config.get('Batch', 'batchaccountkey')
|
|
batch_account_name = global_config.get('Batch', 'batchaccountname')
|
|
batch_service_url = global_config.get('Batch', 'batchserviceurl')
|
|
|
|
# Set up SharedKeyCredentials
|
|
credentials = batch_auth.SharedKeyCredentials(
|
|
batch_account_name,
|
|
batch_account_key)
|
|
|
|
# Set up Batch Client
|
|
batch_client = batch.BatchServiceClient(
|
|
credentials,
|
|
base_url=batch_service_url)
|
|
|
|
# Set retry policy
|
|
batch_client.config.retry_policy.retries = 5
|
|
|
|
# get ssh command
|
|
sparklib.ssh_app(
|
|
batch_client,
|
|
pool_id = _pool_id,
|
|
app_id = _app_id,
|
|
username = _username,
|
|
password = _password)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|