From 8edf46ade45aa93ab6bcc6f0e72c9a4dcfdad022 Mon Sep 17 00:00:00 2001 From: Jon Gallant Date: Tue, 27 Feb 2018 10:43:10 +0100 Subject: [PATCH] azure command message cleanup. sub select by starts_with --- docker/linux/Dockerfile.dev.expanded | 0 docker/linux/Dockerfile.expanded | 2 +- iotedgedev/azurecli.py | 165 ++++++++++++++++----------- iotedgedev/cli.py | 44 +++++-- iotedgedev/envvars.py | 3 + iotedgedev/output.py | 16 ++- iotedgedev/utility.py | 7 +- tests/test_iotedgedev.py | 6 +- 8 files changed, 158 insertions(+), 85 deletions(-) delete mode 100644 docker/linux/Dockerfile.dev.expanded diff --git a/docker/linux/Dockerfile.dev.expanded b/docker/linux/Dockerfile.dev.expanded deleted file mode 100644 index e69de29..0000000 diff --git a/docker/linux/Dockerfile.expanded b/docker/linux/Dockerfile.expanded index b3a92cd..e9021ef 100644 --- a/docker/linux/Dockerfile.expanded +++ b/docker/linux/Dockerfile.expanded @@ -1,2 +1,2 @@ -FROM jongallant/iotedgedev-deps:0.62.0-linux +FROM jongallant/iotedgedev-deps:0.63.0-linux RUN pip --no-cache-dir install -U azure-iot-edge-dev-tool \ No newline at end of file diff --git a/iotedgedev/azurecli.py b/iotedgedev/azurecli.py index 8c7b1e7..a8db9b4 100644 --- a/iotedgedev/azurecli.py +++ b/iotedgedev/azurecli.py @@ -23,26 +23,38 @@ class AzureCli: def is_posix(self): self.envvars.is_posix() - def prepare_az_cli_args(self, args): + def prepare_az_cli_args(self, args, suppress_output=False): + if suppress_output: + args.extend(["--query", "\"[?n]|[0]\""]) + az_args = ["az"]+args if self.is_posix(): return [" ".join(az_args)] return az_args - def invoke_az_cli_outproc(self, args, error_message=None, stdout_io=None, stderr_io=None): + def invoke_az_cli_outproc(self, args, error_message=None, stdout_io=None, stderr_io=None, suppress_output=False): try: + if stdout_io or stderr_io: process = subprocess.Popen(self.prepare_az_cli_args( - args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=not self.envvars.is_posix()) - stdout_data, stderr_data = process.communicate() - if stdout_io and stdout_data != "": - stdout_io.writelines(self.decode(stdout_data)) - if stderr_io and stderr_data != "": - stderr_io.writelines(self.decode(stderr_data)) + args, suppress_output), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=not self.envvars.is_posix()) else: - process = subprocess.Popen( - self.prepare_az_cli_args(args), shell=not self.envvars.is_posix()) - process.communicate() + process = subprocess.Popen(self.prepare_az_cli_args( + args, suppress_output), shell=not self.envvars.is_posix()) + + stdout_data, stderr_data = process.communicate() + + if stderr_data and "400" in stderr_data: + self.output.info( + "Your Azure CLI session has expired. Please re-run iotedgedev azure --setup to refresh your credentials.") + self.logout() + sys.exit() + + if stdout_io and stdout_data != "": + stdout_io.writelines(self.decode(stdout_data)) + + if stderr_io and stderr_data != "": + stderr_io.writelines(self.decode(stderr_data)) if process.returncode != 0: if error_message: @@ -50,6 +62,9 @@ class AzureCli: self.output.line() return False + if not stdout_io and not stderr_io: + self.output.line() + except Exception as e: if error_message: self.output.error(error_message) @@ -57,8 +72,6 @@ class AzureCli: self.output.line() return False - self.output.line() - return True def invoke_az_cli(self, args, error_message=None, stdout_io=None): @@ -79,20 +92,19 @@ class AzureCli: return True def add_extension(self, name): - self.output.header(f("Adding extension {name}")) - return self.invoke_az_cli_outproc(["extension", "add", "--name", name, "--yes"], - f("Error while adding extension {name}.")) + f("Error while adding extension {name}."), suppress_output=True) def extension_exists(self, name): - self.output.header(f("Checking for extension {name}")) - return self.invoke_az_cli_outproc(["extension", "show", "--name", name, "--output", "table"], - f("Error while checking for extension {name}.")) + f("Error while checking for extension {name}."), suppress_output=True) def user_has_logged_in(self): - self.output.header("Checking for cached credentials") + + self.output.header("AUTHENTICATION") + + self.output.status(f("Retrieving Azure CLI credentials from cache...")) with output_io_cls() as io: result = self.invoke_az_cli_outproc( @@ -100,6 +112,7 @@ class AzureCli: if result: try: + self.output.prompt("Azure CLI credentials found.") out_string = io.getvalue() data = json.loads(out_string) return data["id"] @@ -107,35 +120,27 @@ class AzureCli: pass self.output.prompt( "Azure CLI credentials not found. Please follow instructions below to login to the Azure CLI.") - self.output.line() return None def login(self, username, password): - self.output.header("Logging in to Azure") return self.invoke_az_cli_outproc(["login", "-u", username, - "-p", password, "-o", "table"], - "Error while trying to login to Azure. Try logging in with the interactive login mode (do not use the --credentials).") + "-p", password], + "Error while trying to login to Azure. Try logging in with the interactive login mode (do not use the --credentials parameter).", suppress_output=True) def login_interactive(self): - self.output.header("Interactive login to Azure") - - return self.invoke_az_cli_outproc(["login", "--o", "table"], - "Error while trying to login to Azure.") + return self.invoke_az_cli_outproc(["login"], + "Error while trying to login to Azure.", suppress_output=True) def logout(self): - self.output.header("Logout from Azure") - return self.invoke_az_cli_outproc(["account", "clear"]) def list_subscriptions(self): - self.output.header("Listing Subscriptions") - - return self.invoke_az_cli_outproc(["account", "list", "--out", "table"], + self.output.status("Retrieving Azure Subscriptions...") + return self.invoke_az_cli_outproc(["account", "list", "--all", "--query", "[].{\"Subscription Name\":name, Id:id}", "--out", "table"], "Error while trying to list Azure subscriptions.") def get_default_subscription(self): - self.output.header("Getting default subscription id") with output_io_cls() as io: result = self.invoke_az_cli_outproc(["account", "show"], @@ -146,14 +151,29 @@ class AzureCli: return data["id"] return '' + def get_subscription_id_starts_with(self, token): + with output_io_cls() as io: + result = self.invoke_az_cli_outproc(["account", "list", "--query", "[?starts_with(@.id,'{0}') || starts_with(@.name,'{0}')] | [0]".format(token)], + "Could not find a subscription that starts with '{0}'".format(token), io) + if result: + out_string = io.getvalue() + if out_string: + data = json.loads(out_string) + return data["id"] + return '' + def set_subscription(self, subscription): - self.output.header("Setting Subscription") + + if len(subscription) < 36: + subscription = self.get_subscription_id_starts_with(subscription) + + self.output.status(f("Setting Subscription to '{subscription}'...")) return self.invoke_az_cli_outproc(["account", "set", "--subscription", subscription], "Error while trying to set Azure subscription.") def resource_group_exists(self, name): - self.output.header(f("Checking if Resource Group {name} exists")) + self.output.status(f("Checking if Resource Group '{name}' exists...")) with output_io_cls() as io: result = self.invoke_az_cli_outproc(["group", "exists", "-n", name, "--debug"], @@ -165,12 +185,12 @@ class AzureCli: return True self.output.prompt(f("Resource Group {name} does not exist.")) - self.output.line() + return False def create_resource_group(self, name, location): - self.output.header( - f("Creating Resource Group {name} at location:{location}")) + self.output.status( + f("Creating Resource Group '{name}' at '{location}'...")) with output_io_cls() as io: @@ -179,15 +199,12 @@ class AzureCli: return result def list_resource_groups(self): - self.output.header("Listing Resource Groups") - - return self.invoke_az_cli_outproc(["group", "list", "--out", "table"], + self.output.header("RESOURCE GROUP") + self.output.status("Retrieving Resource Groups...") + return self.invoke_az_cli_outproc(["group", "list", "--query", "[].{\"Resource Group\":name, Location:location}", "--out", "table"], "Could not list the Resource Groups.") def get_free_iothub(self): - self.output.header( - f("Checking if an F1 (free) IoT Hub exists in the subscription")) - with output_io_cls() as io: result = self.invoke_az_cli_outproc(["iot", "hub", "list"], @@ -200,16 +217,27 @@ class AzureCli: return (iot["name"], iot["resourceGroup"]) return (None, None) - def list_iot_hubs(self, resource_group): - self.output.header( - f("Listing IoT Hubs in {resource_group}")) + def get_first_iothub(self, resource_group): - return self.invoke_az_cli_outproc(["iot", "hub", "list", "--resource-group", resource_group, "--out", "table"], - f("Could not list the IoT Hubs in {resource_group}.")) + with output_io_cls() as io: + result = self.invoke_az_cli_outproc(["iot", "hub", "list", "--resource-group", resource_group, "--query", "[0]"], f("Could not get first IoT Hub."), io) + + if result: + out_string = io.getvalue() + if out_string: + data = json.loads(out_string) + return data["name"] + return '' + + def list_iot_hubs(self, resource_group): + self.output.header("IOT HUB") + self.output.status(f("Retrieving IoT Hubs in '{resource_group}'...")) + + return self.invoke_az_cli_outproc(["iot", "hub", "list", "--resource-group", resource_group, "--query", "[].{\"IoT Hub\":name}", "--out", "table"], f("Could not list the IoT Hubs in {resource_group}.")) def iothub_exists(self, value, resource_group): - self.output.header( - f("Checking if {value} IoT Hub exists in {resource_group}")) + self.output.status( + f("Checking if '{value}' IoT Hub exists...")) with output_io_cls() as io: @@ -218,12 +246,11 @@ class AzureCli: if not result: self.output.prompt( f("Could not locate the {value} in {resource_group}.")) - self.output.line() return result def create_iothub(self, value, resource_group, sku): - self.output.header( - f("Creating {value} in {resource_group} with sku {sku}")) + self.output.status( + f("Creating '{value}' in '{resource_group}' with '{sku}' sku...")) with output_io_cls() as io: with output_io_cls() as error_io: @@ -231,7 +258,7 @@ class AzureCli: "Creating IoT Hub. Please wait as this could take a few minutes to complete...") result = self.invoke_az_cli_outproc(["iot", "hub", "create", "--name", value, "--resource-group", - resource_group, "--sku", sku, "--out", "table"], + resource_group, "--sku", sku, "--query", "[].{\"IoT Hub\":name}", "--out", "table"], f("Could not create the IoT Hub {value} in {resource_group} with sku {sku}."), stdout_io=io, stderr_io=error_io) if not result and error_io.getvalue(): self.output.error(error_io.getvalue()) @@ -242,8 +269,8 @@ class AzureCli: return result def get_iothub_connection_string(self, value, resource_group): - self.output.header( - f("Getting the connection string for {value} in {resource_group} ")) + self.output.status( + f("Retrieving '{value}' connection string...")) with output_io_cls() as io: result = self.invoke_az_cli_outproc(["iot", "hub", "show-connection-string", "--hub-name", value, @@ -256,8 +283,8 @@ class AzureCli: return '' def edge_device_exists(self, value, iothub, resource_group): - self.output.header( - f("Checking if {value} device exists in {iothub} IoT Hub in {resource_group}")) + self.output.status( + f("Checking if '{value}' device exists in '{iothub}'...")) with output_io_cls() as io: result = self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "show", "--device-id", value, "--hub-name", iothub, @@ -265,28 +292,28 @@ class AzureCli: if not result: self.output.prompt( f("Could not locate the {value} device in {iothub} IoT Hub in {resource_group}.")) - self.output.line() return result def list_edge_devices(self, iothub): - self.output.header( - f("Listing edge devices in {iothub} IoT Hub")) + self.output.header("EDGE DEVICE") + self.output.status( + f("Retrieving edge devices in '{iothub}'...")) return self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "list", "--hub-name", iothub, - "--edge-enabled", "--output", "table"], + "--edge-enabled", "--query", "[].{\"Device Id\":deviceId}", "--output", "table"], f("Could not list the edge devices in {iothub} IoT Hub.")) def create_edge_device(self, value, iothub, resource_group): - self.output.header( - f("Creating {value} edge device in {iothub} IoT Hub in {resource_group}")) + self.output.status( + f("Creating '{value}' edge device in '{iothub}'...")) return self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "create", "--device-id", value, "--hub-name", iothub, - "--resource-group", resource_group, "--edge-enabled", "--output", "table"], + "--resource-group", resource_group, "--edge-enabled", "--query", "[].{\"Device Id\":deviceId}", "--output", "table"], f("Could not locate the {value} device in {iothub} IoT Hub in {resource_group}.")) def get_device_connection_string(self, value, iothub, resource_group): - self.output.header( - f("Getting the connection string for {value} edge device in {iothub} IoT Hub in {resource_group}")) + self.output.status( + f("Retrieving '{value}' connection string...")) with output_io_cls() as io: result = self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "show-connection-string", "--device-id", value, "--hub-name", iothub, diff --git a/iotedgedev/cli.py b/iotedgedev/cli.py index 22a7d59..33bff24 100644 --- a/iotedgedev/cli.py +++ b/iotedgedev/cli.py @@ -22,6 +22,7 @@ output = Output() envvars = EnvVars(output) azure_cli = AzureCli(output, envvars) default_subscriptionId = None +azure_cli_processing_complete = False CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @@ -77,13 +78,18 @@ def iothub(monitor_events): def validate_option(ctx, param, value): global default_subscriptionId + global azure_cli_processing_complete if param.name == "credentials": if value and value[0] and value[1]: + output.param("CREDENTIALS", value, "Setting Credentials...", azure_cli_processing_complete) + if not azure_cli.login(*value): sys.exit() if param.name == "subscription": + output.param("SUBSCRIPTION", value, f("Setting Subscription to '{value}'..."), azure_cli_processing_complete) + # first verify that we have an existing auth token in cache, otherwise login using interactive if not default_subscriptionId: default_subscriptionId = azure_cli.user_has_logged_in() @@ -97,9 +103,14 @@ def validate_option(ctx, param, value): if param.name == "resource_group_location": + + output.param("RESOURCE GROUP LOCATION", value, f("Setting Resource Group Location to '{value}'..."), azure_cli_processing_complete) + envvars.RESOURCE_GROUP_LOCATION = value if param.name == "resource_group_name": + output.param("RESOURCE GROUP NAME", value, f("Setting Resource Group Name to '{value}'..."), azure_cli_processing_complete) + envvars.RESOURCE_GROUP_NAME = value if not azure_cli.resource_group_exists(value): if not azure_cli.create_resource_group(value, envvars.RESOURCE_GROUP_LOCATION): @@ -107,9 +118,12 @@ def validate_option(ctx, param, value): f('Could not find Resource Group {value}')) if param.name == "iothub_sku": + + output.param("IOT HUB SKU", value, f("Setting IoT Hub SKU to '{value}'..."), azure_cli_processing_complete) envvars.IOTHUB_SKU = value if param.name == "iothub_name": + output.param("IOT HUB", value, f("Setting IoT Hub to '{value}'..."), azure_cli_processing_complete) envvars.IOTHUB_NAME = value if not azure_cli.extension_exists("azure-cli-iot-ext"): azure_cli.add_extension("azure-cli-iot-ext") @@ -135,12 +149,15 @@ def validate_option(ctx, param, value): f('Could not create IoT Hub {value} in {envvars.RESOURCE_GROUP_NAME}')) if param.name == "edge_device_id": + output.param("EDGE DEVICE", value, f("Setting Edge Device to '{value}'..."), azure_cli_processing_complete) + envvars.EDGE_DEVICE_ID = value if not azure_cli.edge_device_exists(value, envvars.IOTHUB_NAME, envvars.RESOURCE_GROUP_NAME): if not azure_cli.create_edge_device(value, envvars.IOTHUB_NAME, envvars.RESOURCE_GROUP_NAME): raise click.BadParameter( f('Could not create IoT Edge Device {value} in {envvars.IOTHUB_NAME} in {envvars.RESOURCE_GROUP_NAME}')) + output.header("CONNECTION STRINGS") envvars.IOTHUB_CONNECTION_STRING = azure_cli.get_iothub_connection_string( envvars.IOTHUB_NAME, envvars.RESOURCE_GROUP_NAME) envvars.DEVICE_CONNECTION_STRING = azure_cli.get_device_connection_string( @@ -152,24 +169,33 @@ def validate_option(ctx, param, value): output.info( f("DEVICE_CONNECTION_STRING=\"{envvars.DEVICE_CONNECTION_STRING}\"")) + azure_cli_processing_complete = True + + output.line() + return value def list_edge_devices_and_set_default(): if not azure_cli.list_edge_devices(envvars.IOTHUB_NAME): sys.exit() - return "iotedgedev-edgedevice-dev" + return "iotedgedev-edgedevice" def list_iot_hubs_and_set_default(): if not azure_cli.list_iot_hubs(envvars.RESOURCE_GROUP_NAME): sys.exit() - subscription_rg_hash = hashlib.sha1((default_subscriptionId + envvars.RESOURCE_GROUP_NAME).encode('utf-8')).hexdigest()[:6] - return "iotedgedev-iothub-dev-" + subscription_rg_hash + + first_iothub = azure_cli.get_first_iothub(envvars.RESOURCE_GROUP_NAME) + if first_iothub: + return first_iothub + else: + subscription_rg_hash = hashlib.sha1((default_subscriptionId + envvars.RESOURCE_GROUP_NAME).encode('utf-8')).hexdigest()[:6] + return "iotedgedev-iothub-" + subscription_rg_hash def list_resource_groups_and_set_default(): if not azure_cli.list_resource_groups(): sys.exit() - return "iotedgedev-rg-dev" + return "iotedgedev-rg" def list_subscriptions_and_set_default(): global default_subscriptionId @@ -180,6 +206,8 @@ def list_subscriptions_and_set_default(): if not default_subscriptionId and not azure_cli.login_interactive(): sys.exit() + output.header("SUBSCRIPTION") + if not azure_cli.list_subscriptions(): sys.exit() default_subscriptionId = azure_cli.get_default_subscription() @@ -205,7 +233,7 @@ def list_subscriptions_and_set_default(): default=lambda: list_subscriptions_and_set_default(), required=True, callback=validate_option, - prompt="The Azure subscription name or id to use", + prompt="Enter the first 3 characters of the Azure subscription name or id to use. Hit Enter to use the default subscription.", help="The Azure subscription name or id to use.") @click.option( '--resource-group-location', @@ -220,7 +248,7 @@ def list_subscriptions_and_set_default(): default=lambda: list_resource_groups_and_set_default(), type=str, callback=validate_option, - prompt="The name of the Resource Group to use or create. Creates a new Resource Group if not found", + prompt="Enter the name of the Resource Group to use or create. Creates a new Resource Group if not found", help="The name of the Resource Group to use or create. Creates a new Resource Group if not found.") @click.option( '--iothub-sku', @@ -235,7 +263,7 @@ def list_subscriptions_and_set_default(): default=lambda: list_iot_hubs_and_set_default(), type=str, callback=validate_option, - prompt='The IoT Hub name to be used. Creates a new IoT Hub if not found', + prompt='Enter the IoT Hub name to be used. Creates a new IoT Hub if not found', help='The IoT Hub name to be used. Creates a new IoT Hub if not found.') @click.option( '--edge-device-id', @@ -243,7 +271,7 @@ def list_subscriptions_and_set_default(): default=lambda: list_edge_devices_and_set_default(), type=str, callback=validate_option, - prompt='The IoT Edge Device Id to be used. Creates a new Edge Device if not found', + prompt='Enter the IoT Edge Device Id to be used. Creates a new Edge Device if not found', help='The IoT Edge Device Id to be used. Creates a new Edge Device if not found.') @click.option( '--update-dotenv', diff --git a/iotedgedev/envvars.py b/iotedgedev/envvars.py index 2df735f..d4bd3ab 100644 --- a/iotedgedev/envvars.py +++ b/iotedgedev/envvars.py @@ -56,6 +56,9 @@ class EnvVars: def check(self): if not self.checked: + + self.output.header("ENVIRONMENT VARIABLES") + self.load_dotenv() try: diff --git a/iotedgedev/output.py b/iotedgedev/output.py index 6398e70..c91b94e 100644 --- a/iotedgedev/output.py +++ b/iotedgedev/output.py @@ -5,6 +5,10 @@ class Output: def info(self, text): click.secho(text, fg='yellow') + def status(self, text): + self.info(text) + self.line() + def prompt(self, text): click.secho(text, fg='white') @@ -13,7 +17,17 @@ class Output: def header(self, text): self.line() - click.secho("======== {0} ========".format(text).upper(), fg='white') + s = "======== {0} ========".format(text).upper() + m = "="*len(s) + click.secho(m, fg='white') + click.secho(s, fg='white') + click.secho(m, fg='white') + self.line() + + def param(self, text, value, status, suppress): + if value and not suppress: + self.header("SETTING " + text) + self.status(status) def footer(self, text): self.info(text.upper()) diff --git a/iotedgedev/utility.py b/iotedgedev/utility.py index f1191f9..8d718c4 100644 --- a/iotedgedev/utility.py +++ b/iotedgedev/utility.py @@ -15,9 +15,10 @@ else: from .moduletype import ModuleType class Utility: - def __init__(self, envvars, output): + def __init__(self, envvars, output, envvars_check=True): self.envvars = envvars - self.envvars.check() + if envvars_check: + self.envvars.check() self.output = output self.config_set = False @@ -65,7 +66,7 @@ class Utility: with open(file, "r") as file: return file.read() - def decode(self, val): + def decode(self, val): return val.decode("utf-8").strip() def get_config_files(self): diff --git a/tests/test_iotedgedev.py b/tests/test_iotedgedev.py index c68c67a..9815999 100644 --- a/tests/test_iotedgedev.py +++ b/tests/test_iotedgedev.py @@ -104,9 +104,9 @@ class TestIotedgedev(unittest.TestCase): 'azure', '--setup', '--credentials', 'username', 'password', '--subscription', '12341234-1234-1234-1234-123412341234', - '--resource-group-name', 'iotedgedev-rg-dev', - '--iothub-name', 'iotedgedev-iothub-dev', - '--edge-device-id', 'iotedgedev-edgedevice-dev', + '--resource-group-name', 'iotedgedev-rg', + '--iothub-name', 'iotedgedev-iothub', + '--edge-device-id', 'iotedgedev-edgedevice', '--update-dotenv' ], az_cli = TestAzureCli(sys.stdout) ) print(result.output)