azure command message cleanup. sub select by starts_with

docker/linux/Dockerfile.expanded

@@ -1,2 +1,2 @@
-FROM jongallant/iotedgedev-deps:0.62.0-linux
+FROM jongallant/iotedgedev-deps:0.63.0-linux
 RUN pip --no-cache-dir install -U azure-iot-edge-dev-tool

iotedgedev/azurecli.py

@@ -23,26 +23,38 @@ class AzureCli:
     def is_posix(self):
         self.envvars.is_posix()
 
-    def prepare_az_cli_args(self, args):
+    def prepare_az_cli_args(self, args, suppress_output=False):
+        if suppress_output:
+            args.extend(["--query", "\"[?n]|[0]\""])
+
         az_args = ["az"]+args
         if self.is_posix():
             return [" ".join(az_args)]
         return az_args
 
-    def invoke_az_cli_outproc(self, args, error_message=None, stdout_io=None, stderr_io=None):
+    def invoke_az_cli_outproc(self, args, error_message=None, stdout_io=None, stderr_io=None, suppress_output=False):
         try:
+
             if stdout_io or stderr_io:
                 process = subprocess.Popen(self.prepare_az_cli_args(
-                    args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=not self.envvars.is_posix())
+                    args, suppress_output), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=not self.envvars.is_posix())
+            else:
+                process = subprocess.Popen(self.prepare_az_cli_args(
+                    args, suppress_output), shell=not self.envvars.is_posix())
+
             stdout_data, stderr_data = process.communicate()
+
+            if stderr_data and "400" in stderr_data:
+                self.output.info(
+                    "Your Azure CLI session has expired. Please re-run iotedgedev azure --setup to refresh your credentials.")
+                self.logout()
+                sys.exit()
+
             if stdout_io and stdout_data != "":
                 stdout_io.writelines(self.decode(stdout_data))
+
             if stderr_io and stderr_data != "":
                 stderr_io.writelines(self.decode(stderr_data))
-            else:
-                process = subprocess.Popen(
-                    self.prepare_az_cli_args(args), shell=not self.envvars.is_posix())
-                process.communicate()
 
             if process.returncode != 0:
                 if error_message:
@@ -50,6 +62,9 @@ class AzureCli:
                     self.output.line()
                 return False
 
+            if not stdout_io and not stderr_io:
+                self.output.line()
+
         except Exception as e:
             if error_message:
                 self.output.error(error_message)
@@ -57,8 +72,6 @@ class AzureCli:
             self.output.line()
             return False
 
-        self.output.line()
-
         return True
 
     def invoke_az_cli(self, args, error_message=None, stdout_io=None):
@@ -79,20 +92,19 @@ class AzureCli:
         return True
 
     def add_extension(self, name):
-        self.output.header(f("Adding extension {name}"))
-
         return self.invoke_az_cli_outproc(["extension", "add", "--name", name,
                                            "--yes"],
-                                          f("Error while adding extension {name}."))
+                                          f("Error while adding extension {name}."), suppress_output=True)
 
     def extension_exists(self, name):
-        self.output.header(f("Checking for extension {name}"))
-
         return self.invoke_az_cli_outproc(["extension", "show", "--name", name, "--output", "table"],
-                                          f("Error while checking for extension {name}."))
+                                          f("Error while checking for extension {name}."), suppress_output=True)
 
     def user_has_logged_in(self):
-        self.output.header("Checking for cached credentials")
+
+        self.output.header("AUTHENTICATION")
+
+        self.output.status(f("Retrieving Azure CLI credentials from cache..."))
 
         with output_io_cls() as io:
             result = self.invoke_az_cli_outproc(
@@ -100,6 +112,7 @@ class AzureCli:
 
             if result:
                 try:
+                    self.output.prompt("Azure CLI credentials found.")
                     out_string = io.getvalue()
                     data = json.loads(out_string)
                     return data["id"]
@@ -107,35 +120,27 @@ class AzureCli:
                     pass
         self.output.prompt(
             "Azure CLI credentials not found. Please follow instructions below to login to the Azure CLI.")
-        self.output.line()
         return None
 
     def login(self, username, password):
-        self.output.header("Logging in to Azure")
 
         return self.invoke_az_cli_outproc(["login", "-u", username,
-                                           "-p",  password, "-o", "table"],
+                                           "-p",  password],
-                                          "Error while trying to login to Azure. Try logging in with the interactive login mode (do not use the --credentials).")
+                                          "Error while trying to login to Azure. Try logging in with the interactive login mode (do not use the --credentials parameter).", suppress_output=True)
 
     def login_interactive(self):
-        self.output.header("Interactive login to Azure")
+        return self.invoke_az_cli_outproc(["login"],
-
+                                          "Error while trying to login to Azure.", suppress_output=True)
-        return self.invoke_az_cli_outproc(["login", "--o", "table"],
-                                          "Error while trying to login to Azure.")
 
     def logout(self):
-        self.output.header("Logout from Azure")
-
         return self.invoke_az_cli_outproc(["account", "clear"])
 
     def list_subscriptions(self):
-        self.output.header("Listing Subscriptions")
+        self.output.status("Retrieving Azure Subscriptions...")
-
+        return self.invoke_az_cli_outproc(["account", "list", "--all", "--query", "[].{\"Subscription Name\":name, Id:id}", "--out", "table"],
-        return self.invoke_az_cli_outproc(["account", "list", "--out", "table"],
                                           "Error while trying to list Azure subscriptions.")
 
     def get_default_subscription(self):
-        self.output.header("Getting default subscription id")
 
         with output_io_cls() as io:
             result = self.invoke_az_cli_outproc(["account", "show"],
@@ -146,14 +151,29 @@ class AzureCli:
                 return data["id"]
         return ''
 
+    def get_subscription_id_starts_with(self, token):
+        with output_io_cls() as io:
+            result = self.invoke_az_cli_outproc(["account", "list", "--query", "[?starts_with(@.id,'{0}') || starts_with(@.name,'{0}')] | [0]".format(token)],
+                                                "Could not find a subscription that starts with '{0}'".format(token), io)
+            if result:
+                out_string = io.getvalue()
+                if out_string:
+                    data = json.loads(out_string)
+                    return data["id"]
+        return ''
+
     def set_subscription(self, subscription):
-        self.output.header("Setting Subscription")
+
+        if len(subscription) < 36:
+            subscription = self.get_subscription_id_starts_with(subscription)
+
+        self.output.status(f("Setting Subscription to '{subscription}'..."))
 
         return self.invoke_az_cli_outproc(["account", "set", "--subscription", subscription],
                                           "Error while trying to set Azure subscription.")
 
     def resource_group_exists(self, name):
-        self.output.header(f("Checking if Resource Group {name} exists"))
+        self.output.status(f("Checking if Resource Group '{name}' exists..."))
 
         with output_io_cls() as io:
             result = self.invoke_az_cli_outproc(["group", "exists", "-n", name, "--debug"],
@@ -165,12 +185,12 @@ class AzureCli:
                     return True
 
         self.output.prompt(f("Resource Group {name} does not exist."))
-        self.output.line()
+
         return False
 
     def create_resource_group(self, name, location):
-        self.output.header(
+        self.output.status(
-            f("Creating Resource Group {name} at location:{location}"))
+            f("Creating Resource Group '{name}' at '{location}'..."))
 
         with output_io_cls() as io:
 
@@ -179,15 +199,12 @@ class AzureCli:
         return result
 
     def list_resource_groups(self):
-        self.output.header("Listing Resource Groups")
+        self.output.header("RESOURCE GROUP")
-
+        self.output.status("Retrieving Resource Groups...")
-        return self.invoke_az_cli_outproc(["group", "list", "--out", "table"],
+        return self.invoke_az_cli_outproc(["group", "list", "--query", "[].{\"Resource Group\":name, Location:location}", "--out", "table"],
                                           "Could not list the Resource Groups.")
 
     def get_free_iothub(self):
-        self.output.header(
-            f("Checking if an F1 (free) IoT Hub exists in the subscription"))
-
         with output_io_cls() as io:
 
             result = self.invoke_az_cli_outproc(["iot", "hub", "list"],
@@ -200,16 +217,27 @@ class AzureCli:
                         return (iot["name"], iot["resourceGroup"])
         return (None, None)
 
-    def list_iot_hubs(self, resource_group):
+    def get_first_iothub(self, resource_group):
-        self.output.header(
-            f("Listing IoT Hubs in {resource_group}"))
 
-        return self.invoke_az_cli_outproc(["iot", "hub", "list", "--resource-group", resource_group, "--out", "table"],
+        with output_io_cls() as io:
-                                          f("Could not list the IoT Hubs in {resource_group}."))
+            result = self.invoke_az_cli_outproc(["iot", "hub", "list", "--resource-group", resource_group, "--query", "[0]"], f("Could not get first IoT Hub."), io)
+
+            if result:
+                out_string = io.getvalue()
+                if out_string:
+                    data = json.loads(out_string)
+                    return data["name"]
+        return ''
+
+    def list_iot_hubs(self, resource_group):
+        self.output.header("IOT HUB")
+        self.output.status(f("Retrieving IoT Hubs in '{resource_group}'..."))
+
+        return self.invoke_az_cli_outproc(["iot", "hub", "list", "--resource-group", resource_group, "--query", "[].{\"IoT Hub\":name}", "--out", "table"], f("Could not list the IoT Hubs in {resource_group}."))
 
     def iothub_exists(self, value, resource_group):
-        self.output.header(
+        self.output.status(
-            f("Checking if {value} IoT Hub exists in {resource_group}"))
+            f("Checking if '{value}' IoT Hub exists..."))
 
         with output_io_cls() as io:
 
@@ -218,12 +246,11 @@ class AzureCli:
         if not result:
             self.output.prompt(
                 f("Could not locate the {value} in {resource_group}."))
-            self.output.line()
         return result
 
     def create_iothub(self, value, resource_group, sku):
-        self.output.header(
+        self.output.status(
-            f("Creating {value} in {resource_group} with sku {sku}"))
+            f("Creating '{value}' in '{resource_group}' with '{sku}' sku..."))
 
         with output_io_cls() as io:
             with output_io_cls() as error_io:
@@ -231,7 +258,7 @@ class AzureCli:
                     "Creating IoT Hub. Please wait as this could take a few minutes to complete...")
 
                 result = self.invoke_az_cli_outproc(["iot", "hub", "create", "--name", value, "--resource-group",
-                                                     resource_group, "--sku", sku, "--out", "table"],
+                                                     resource_group, "--sku", sku, "--query", "[].{\"IoT Hub\":name}", "--out", "table"],
                                                     f("Could not create the IoT Hub {value} in {resource_group} with sku {sku}."), stdout_io=io, stderr_io=error_io)
                 if not result and error_io.getvalue():
                     self.output.error(error_io.getvalue())
@@ -242,8 +269,8 @@ class AzureCli:
         return result
 
     def get_iothub_connection_string(self, value, resource_group):
-        self.output.header(
+        self.output.status(
-            f("Getting the connection string for {value} in {resource_group} "))
+            f("Retrieving '{value}' connection string..."))
 
         with output_io_cls() as io:
             result = self.invoke_az_cli_outproc(["iot", "hub", "show-connection-string", "--hub-name", value,
@@ -256,8 +283,8 @@ class AzureCli:
         return ''
 
     def edge_device_exists(self, value, iothub, resource_group):
-        self.output.header(
+        self.output.status(
-            f("Checking if {value} device exists in {iothub} IoT Hub in {resource_group}"))
+            f("Checking if '{value}' device exists in '{iothub}'..."))
 
         with output_io_cls() as io:
             result = self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "show", "--device-id", value, "--hub-name", iothub,
@@ -265,28 +292,28 @@ class AzureCli:
         if not result:
             self.output.prompt(
                 f("Could not locate the {value} device in {iothub} IoT Hub in {resource_group}."))
-            self.output.line()
         return result
 
     def list_edge_devices(self, iothub):
-        self.output.header(
+        self.output.header("EDGE DEVICE")
-            f("Listing edge devices in {iothub} IoT Hub"))
+        self.output.status(
+            f("Retrieving edge devices in '{iothub}'..."))
 
         return self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "list", "--hub-name", iothub,
-                                           "--edge-enabled", "--output", "table"],
+                                           "--edge-enabled", "--query", "[].{\"Device Id\":deviceId}", "--output", "table"],
                                           f("Could not list the edge devices  in {iothub} IoT Hub."))
 
     def create_edge_device(self, value, iothub, resource_group):
-        self.output.header(
+        self.output.status(
-            f("Creating {value} edge device in {iothub} IoT Hub in {resource_group}"))
+            f("Creating '{value}' edge device in '{iothub}'..."))
 
         return self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "create", "--device-id", value, "--hub-name", iothub,
-                                           "--resource-group", resource_group, "--edge-enabled", "--output", "table"],
+                                           "--resource-group", resource_group, "--edge-enabled", "--query", "[].{\"Device Id\":deviceId}", "--output", "table"],
                                           f("Could not locate the {value} device in {iothub} IoT Hub in {resource_group}."))
 
     def get_device_connection_string(self, value, iothub, resource_group):
-        self.output.header(
+        self.output.status(
-            f("Getting the connection string for {value} edge device in {iothub} IoT Hub in {resource_group}"))
+            f("Retrieving '{value}' connection string..."))
 
         with output_io_cls() as io:
             result = self.invoke_az_cli_outproc(["iot", "hub", "device-identity", "show-connection-string", "--device-id", value, "--hub-name", iothub,

iotedgedev/cli.py

@@ -22,6 +22,7 @@ output = Output()
 envvars = EnvVars(output)
 azure_cli = AzureCli(output, envvars)
 default_subscriptionId = None
+azure_cli_processing_complete = False
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -77,13 +78,18 @@ def iothub(monitor_events):
 
 def validate_option(ctx, param, value):
     global default_subscriptionId
+    global azure_cli_processing_complete
 
     if param.name == "credentials":
         if value and value[0] and value[1]:
+            output.param("CREDENTIALS", value, "Setting Credentials...", azure_cli_processing_complete)
+
             if not azure_cli.login(*value):
                 sys.exit()
 
     if param.name == "subscription":
+        output.param("SUBSCRIPTION", value, f("Setting Subscription to '{value}'..."), azure_cli_processing_complete)
+
         # first verify that we have an existing auth token in cache, otherwise login using interactive
         if not default_subscriptionId:
             default_subscriptionId = azure_cli.user_has_logged_in()
@@ -97,9 +103,14 @@ def validate_option(ctx, param, value):
 
 
     if param.name == "resource_group_location":
+        
+        output.param("RESOURCE GROUP LOCATION", value, f("Setting Resource Group Location to '{value}'..."), azure_cli_processing_complete)
+
         envvars.RESOURCE_GROUP_LOCATION = value
 
     if param.name == "resource_group_name":
+        output.param("RESOURCE GROUP NAME", value, f("Setting Resource Group Name to '{value}'..."), azure_cli_processing_complete)
+
         envvars.RESOURCE_GROUP_NAME = value
         if not azure_cli.resource_group_exists(value):
             if not azure_cli.create_resource_group(value, envvars.RESOURCE_GROUP_LOCATION):
@@ -107,9 +118,12 @@ def validate_option(ctx, param, value):
                     f('Could not find Resource Group {value}'))
 
     if param.name == "iothub_sku":
+
+        output.param("IOT HUB SKU", value, f("Setting IoT Hub SKU to '{value}'..."), azure_cli_processing_complete)
         envvars.IOTHUB_SKU = value
 
     if param.name == "iothub_name":
+        output.param("IOT HUB", value, f("Setting IoT Hub to '{value}'..."), azure_cli_processing_complete)
         envvars.IOTHUB_NAME = value
         if not azure_cli.extension_exists("azure-cli-iot-ext"):
             azure_cli.add_extension("azure-cli-iot-ext")
@@ -135,12 +149,15 @@ def validate_option(ctx, param, value):
                     f('Could not create IoT Hub {value} in {envvars.RESOURCE_GROUP_NAME}'))
 
     if param.name == "edge_device_id":
+        output.param("EDGE DEVICE", value, f("Setting Edge Device to '{value}'..."), azure_cli_processing_complete)
+
         envvars.EDGE_DEVICE_ID = value
         if not azure_cli.edge_device_exists(value, envvars.IOTHUB_NAME, envvars.RESOURCE_GROUP_NAME):
             if not azure_cli.create_edge_device(value, envvars.IOTHUB_NAME, envvars.RESOURCE_GROUP_NAME):
                 raise click.BadParameter(
                     f('Could not create IoT Edge Device {value} in {envvars.IOTHUB_NAME} in {envvars.RESOURCE_GROUP_NAME}'))
 
+        output.header("CONNECTION STRINGS")
         envvars.IOTHUB_CONNECTION_STRING = azure_cli.get_iothub_connection_string(
             envvars.IOTHUB_NAME, envvars.RESOURCE_GROUP_NAME)
         envvars.DEVICE_CONNECTION_STRING = azure_cli.get_device_connection_string(
@@ -152,24 +169,33 @@ def validate_option(ctx, param, value):
             output.info(
                 f("DEVICE_CONNECTION_STRING=\"{envvars.DEVICE_CONNECTION_STRING}\""))
 
+        azure_cli_processing_complete = True
+
+        output.line()
+
     return value
 
 def list_edge_devices_and_set_default():
     if not azure_cli.list_edge_devices(envvars.IOTHUB_NAME):
         sys.exit()
-    return "iotedgedev-edgedevice-dev"
+    return "iotedgedev-edgedevice"
     
 def list_iot_hubs_and_set_default():
     if not azure_cli.list_iot_hubs(envvars.RESOURCE_GROUP_NAME):
         sys.exit()
+
+    first_iothub = azure_cli.get_first_iothub(envvars.RESOURCE_GROUP_NAME)
+    if first_iothub:
+        return first_iothub
+    else:
         subscription_rg_hash = hashlib.sha1((default_subscriptionId + envvars.RESOURCE_GROUP_NAME).encode('utf-8')).hexdigest()[:6]
-    return "iotedgedev-iothub-dev-" + subscription_rg_hash
+        return "iotedgedev-iothub-" + subscription_rg_hash
 
 
 def list_resource_groups_and_set_default():
     if not azure_cli.list_resource_groups():
         sys.exit()
-    return "iotedgedev-rg-dev"
+    return "iotedgedev-rg"
 
 def list_subscriptions_and_set_default():
     global default_subscriptionId
@@ -180,6 +206,8 @@ def list_subscriptions_and_set_default():
         if not default_subscriptionId and not azure_cli.login_interactive():
             sys.exit()
 
+    output.header("SUBSCRIPTION")
+
     if not azure_cli.list_subscriptions():
         sys.exit()
     default_subscriptionId = azure_cli.get_default_subscription()
@@ -205,7 +233,7 @@ def list_subscriptions_and_set_default():
     default=lambda: list_subscriptions_and_set_default(),
     required=True,
     callback=validate_option,
-    prompt="The Azure subscription name or id to use",
+    prompt="Enter the first 3 characters of the Azure subscription name or id to use. Hit Enter to use the default subscription.",
     help="The Azure subscription name or id to use.")
 @click.option(
     '--resource-group-location',
@@ -220,7 +248,7 @@ def list_subscriptions_and_set_default():
     default=lambda: list_resource_groups_and_set_default(),
     type=str,
     callback=validate_option,
-    prompt="The name of the Resource Group to use or create. Creates a new Resource Group if not found",
+    prompt="Enter the name of the Resource Group to use or create. Creates a new Resource Group if not found",
     help="The name of the Resource Group to use or create. Creates a new Resource Group if not found.")
 @click.option(
     '--iothub-sku',
@@ -235,7 +263,7 @@ def list_subscriptions_and_set_default():
     default=lambda: list_iot_hubs_and_set_default(),
     type=str,
     callback=validate_option,
-    prompt='The IoT Hub name to be used. Creates a new IoT Hub if not found',
+    prompt='Enter the IoT Hub name to be used. Creates a new IoT Hub if not found',
     help='The IoT Hub name to be used. Creates a new IoT Hub if not found.')
 @click.option(
     '--edge-device-id',
@@ -243,7 +271,7 @@ def list_subscriptions_and_set_default():
     default=lambda: list_edge_devices_and_set_default(),
     type=str,
     callback=validate_option,
-    prompt='The IoT Edge Device Id to be used. Creates a new Edge Device if not found',
+    prompt='Enter the IoT Edge Device Id to be used. Creates a new Edge Device if not found',
     help='The IoT Edge Device Id to be used. Creates a new Edge Device if not found.')
 @click.option(
     '--update-dotenv',

iotedgedev/envvars.py

@@ -56,6 +56,9 @@ class EnvVars:
 
     def check(self):
         if not self.checked:
+            
+            self.output.header("ENVIRONMENT VARIABLES")
+            
             self.load_dotenv()
 
             try:

iotedgedev/output.py

@@ -5,6 +5,10 @@ class Output:
     def info(self, text):
         click.secho(text, fg='yellow')
 
+    def status(self, text):
+        self.info(text)
+        self.line()
+
     def prompt(self, text):
         click.secho(text, fg='white')
 
@@ -13,7 +17,17 @@ class Output:
 
     def header(self, text):
         self.line()
-        click.secho("======== {0} ========".format(text).upper(), fg='white')
+        s = "======== {0} ========".format(text).upper()
+        m = "="*len(s)
+        click.secho(m, fg='white')
+        click.secho(s, fg='white')
+        click.secho(m, fg='white')
+        self.line()
+
+    def param(self, text, value, status, suppress):
+        if value and not suppress:
+            self.header("SETTING " + text)
+            self.status(status)
 
     def footer(self, text):
         self.info(text.upper())

iotedgedev/utility.py

@@ -15,8 +15,9 @@ else:
 from .moduletype import ModuleType
 
 class Utility:
-    def __init__(self, envvars, output):
+    def __init__(self, envvars, output, envvars_check=True):
         self.envvars = envvars
+        if envvars_check:
             self.envvars.check()
         self.output = output
         self.config_set = False

tests/test_iotedgedev.py

@@ -104,9 +104,9 @@ class TestIotedgedev(unittest.TestCase):
                                'azure', '--setup',
                                '--credentials', 'username', 'password',
                                '--subscription', '12341234-1234-1234-1234-123412341234',
-                               '--resource-group-name', 'iotedgedev-rg-dev',
+                               '--resource-group-name', 'iotedgedev-rg',
-                               '--iothub-name', 'iotedgedev-iothub-dev',
+                               '--iothub-name', 'iotedgedev-iothub',
-                               '--edge-device-id', 'iotedgedev-edgedevice-dev',
+                               '--edge-device-id', 'iotedgedev-edgedevice',
                                '--update-dotenv'
                                ], az_cli = TestAzureCli(sys.stdout) )
         print(result.output)