Add boot diagnostics support

- Move to Ubuntu 18.04-LTS stable
- Enables serial console access
- Resolves #193

convoy/remotefs.py

@@ -548,7 +548,7 @@ def _create_virtual_machine(
                 ImageReference(
                     publisher='Canonical',
                     offer='UbuntuServer',
-                    sku='18.04-DAILY-LTS',  # TODO move to stable after release
+                    sku='18.04-LTS',
                     version='latest',
                 ),
                 data_disks=data_disks,
@@ -574,6 +574,17 @@ def _create_virtual_machine(
                     ),
                 ),
             ),
+            diagnostics_profile=compute_client.virtual_machines.models.
+            DiagnosticsProfile(
+                boot_diagnostics=compute_client.virtual_machines.models.
+                BootDiagnostics(
+                    enabled=True,
+                    storage_uri='https://{}.blob.{}'.format(
+                        storage.get_storageaccount(),
+                        storage.get_storageaccount_endpoint(),
+                    ),
+                ),
+            ),
         ),
     )
 
@@ -1789,6 +1800,8 @@ def delete_storage_cluster(
                     compute_client, network_client, rfs, vm)
             resources[i] = {
                 'vm': vm.name,
+                'arm_id': vm.id,
+                'id': vm.vm_id,
                 'as': None,
                 'nic': nic,
                 'pip': pip,
@@ -1939,6 +1952,16 @@ def delete_storage_cluster(
             compute_client, rfs.storage_cluster.resource_group, as_name)
         logger.info('availability set {} deleted'.format(as_name))
     deleted.clear()
+    # delete boot diagnostics storage containers
+    for key in resources:
+        try:
+            vm_name = resources[key]['vm']
+            vm_id = resources[key]['id']
+        except KeyError:
+            pass
+        else:
+            storage.delete_storage_containers_remotefs_boot_diagnostics(
+                blob_client, vm_name, vm_id)
     # delete storage container
     storage.delete_storage_containers_remotefs(blob_client)
     # wait for all async ops to complete

convoy/storage.py

@@ -33,6 +33,7 @@ from builtins import (  # noqa
 import datetime
 import hashlib
 import logging
+import re
 # non-stdlib imports
 import azure.common
 import azure.cosmosdb.table as azuretable
@@ -618,7 +619,27 @@ def delete_storage_containers_remotefs(blob_client):
     """
     contname = _STORAGE_CONTAINERS['blob_remotefs']
     logger.info('deleting container: {}'.format(contname))
+    try:
         blob_client.delete_container(contname)
+    except azure.common.AzureMissingResourceHttpError:
+        logger.warning('container not found: {}'.format(contname))
+
+
+def delete_storage_containers_remotefs_boot_diagnostics(
+        blob_client, vm_name, vm_id):
+    # type: (azureblob.BlockBlobService, str, str) -> None
+    """Delete storage containers used for remotefs bootdiagnostics
+    :param azure.storage.blob.BlockBlobService blob_client: blob client
+    :param str vm_name: vm name
+    :param str vm_id: vm id
+    """
+    contname = 'bootdiagnostics-{}-{}'.format(
+        re.sub('[\W_]+', '', vm_name)[0:9], vm_id)
+    logger.info('deleting container: {}'.format(contname))
+    try:
+        blob_client.delete_container(contname)
+    except azure.common.AzureMissingResourceHttpError:
+        logger.warning('container not found: {}'.format(contname))
 
 
 def cleanup_with_del_pool(blob_client, table_client, config, pool_id=None):

docs/65-batch-shipyard-remote-fs.md

@@ -52,6 +52,7 @@ machine to disk mapping
 if requested
 * Automatic placement in an availability set for GlusterFS virtual machines
 * Support for [accelerated networking](https://docs.microsoft.com/azure/virtual-network/create-vm-accelerated-networking-cli)
+* Automatic boot diagnostics enablement and support for serial console access
 * Automatic SSH keypair provisioning and setup for all file servers in
 storage cluster
 * Configuration-driven data ingress support via scp and rsync+ssh, including