Airlock - Adding missing blocked storage (#2163)

* Adding missing blocked storage
* opening the Scan result Event grid topic to public access so malware scanning can deliver messages
* adding missing scan_result topic subscription

airlock_processor/_version.py

@@ -1 +1 @@
-__version__ = "0.0.9"
+__version__ = "0.0.10"

airlock_processor/shared_code/constants.py

@@ -16,7 +16,7 @@ STORAGE_ACCOUNT_NAME_EXPORT_INTERNAL = "stalexintws"
 STORAGE_ACCOUNT_NAME_EXPORT_INPROGRESS = "stalexipws"
 STORAGE_ACCOUNT_NAME_EXPORT_APPROVED = "stalexapp"
 STORAGE_ACCOUNT_NAME_EXPORT_REJECTED = "stalexrejws"
-STORAGE_ACCOUNT_NAME_EXPORT_BLOCKED = "stalimblockedws"
+STORAGE_ACCOUNT_NAME_EXPORT_BLOCKED = "stalexblockedws"
 
 # Stages
 STAGE_DRAFT = "draft"

templates/core/terraform/airlock/eventgrid_topics.tf

@@ -197,6 +197,53 @@ resource "null_resource" "wait_for_import_rejected_blob_created" {
   depends_on = [azurerm_eventgrid_system_topic.import_rejected_blob_created]
 }
 
+resource "azurerm_eventgrid_system_topic" "import_blocked_blob_created" {
+  name                   = local.import_blocked_sys_topic_name
+  location               = var.location
+  resource_group_name    = var.resource_group_name
+  source_arm_resource_id = azurerm_storage_account.sa_import_blocked.id
+  topic_type             = "Microsoft.Storage.StorageAccounts"
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  tags = merge(var.tre_core_tags, {
+    Publishers = "airlock;import-blocked-sa"
+  })
+
+  depends_on = [
+    azurerm_storage_account.sa_import_blocked,
+    null_resource.wait_for_import_rejected_blob_created
+  ]
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_role_assignment" "servicebus_sender_import_blocked_blob_created" {
+  scope                = var.airlock_servicebus.id
+  role_definition_name = "Azure Service Bus Data Sender"
+  principal_id         = azurerm_eventgrid_system_topic.import_blocked_blob_created.identity.0.principal_id
+
+  depends_on = [
+    azurerm_eventgrid_system_topic.import_blocked_blob_created
+  ]
+}
+
+# TEMPPORARY MITIGATION. Should be removed with https://github.com/microsoft/AzureTRE/issues/2164
+resource "null_resource" "wait_for_import_blocked_blob_created" {
+  provisioner "local-exec" {
+    command    = "bash -c \"sleep 60s\""
+    on_failure = fail
+  }
+
+  triggers = {
+    always_run = timestamp()
+  }
+
+  depends_on = [azurerm_eventgrid_system_topic.import_blocked_blob_created]
+}
+
 
 resource "azurerm_eventgrid_system_topic" "export_approved_blob_created" {
   name                   = local.export_approved_sys_topic_name
@@ -215,7 +262,7 @@ resource "azurerm_eventgrid_system_topic" "export_approved_blob_created" {
 
   depends_on = [
     azurerm_storage_account.sa_export_approved,
-    null_resource.wait_for_import_rejected_blob_created
+    null_resource.wait_for_import_blocked_blob_created
   ]
 
   lifecycle { ignore_changes = [tags] }
@@ -234,10 +281,12 @@ resource "azurerm_role_assignment" "servicebus_sender_export_approved_blob_creat
 
 # Custom topic (for scanning)
 resource "azurerm_eventgrid_topic" "scan_result" {
-  name                          = local.scan_result_topic_name
-  location                      = var.location
-  resource_group_name           = var.resource_group_name
-  public_network_access_enabled = false
+  name                = local.scan_result_topic_name
+  location            = var.location
+  resource_group_name = var.resource_group_name
+
+  # Must enable public access so the malware scanning service can report back
+  public_network_access_enabled = true
 
   identity {
     type = "SystemAssigned"
@@ -314,6 +363,22 @@ resource "azurerm_eventgrid_event_subscription" "status_changed" {
   ]
 }
 
+resource "azurerm_eventgrid_event_subscription" "scan_result" {
+  name  = local.scan_result_eventgrid_subscription_name
+  scope = azurerm_eventgrid_topic.scan_result.id
+
+  service_bus_queue_endpoint_id = azurerm_servicebus_queue.scan_result.id
+
+  delivery_identity {
+    type = "SystemAssigned"
+  }
+
+  depends_on = [
+    azurerm_eventgrid_topic.scan_result,
+    azurerm_role_assignment.servicebus_sender_scan_result
+  ]
+}
+
 resource "azurerm_eventgrid_event_subscription" "import_inprogress_blob_created" {
   name  = local.import_inprogress_eventgrid_subscription_name
   scope = azurerm_storage_account.sa_import_in_progress.id
@@ -348,6 +413,25 @@ resource "azurerm_eventgrid_event_subscription" "import_rejected_blob_created" {
   ]
 }
 
+
+resource "azurerm_eventgrid_event_subscription" "import_blocked_blob_created" {
+  name  = local.import_blocked_eventgrid_subscription_name
+  scope = azurerm_storage_account.sa_import_blocked.id
+
+  service_bus_topic_endpoint_id = azurerm_servicebus_topic.blob_created.id
+
+  delivery_identity {
+    type = "SystemAssigned"
+  }
+
+  # Todo add Dead_letter
+
+  depends_on = [
+    azurerm_eventgrid_system_topic.import_blocked_blob_created,
+    azurerm_role_assignment.servicebus_sender_import_blocked_blob_created
+  ]
+}
+
 resource "azurerm_eventgrid_event_subscription" "export_approved_blob_created" {
   name  = local.export_approved_eventgrid_subscription_name
   scope = azurerm_storage_account.sa_export_approved.id

templates/core/terraform/airlock/locals.tf

@@ -1,11 +1,13 @@
 locals {
   # STorage AirLock EXternal
   import_external_storage_name = lower(replace("stalimex${var.tre_id}", "-", ""))
-  # STorage AirLock InProgress IMport
+  # STorage AirLock IMport InProgress
   import_in_progress_storage_name = lower(replace("stalimip${var.tre_id}", "-", ""))
-  # STorage AirLock REJected IMport
+  # STorage AirLock IMport REJected
   import_rejected_storage_name = lower(replace("stalimrej${var.tre_id}", "-", ""))
-  # STorage AirLock APProved EXPort
+  # STorage AirLock IMport BLOCKED
+  import_blocked_storage_name = lower(replace("stalimblocked${var.tre_id}", "-", ""))
+  # STorage AirLock EXPort APProved
   export_approved_storage_name = lower(replace("stalexapp${var.tre_id}", "-", ""))
 
   # Due to the following issue and Azure not liking delete and immediate recreate under the same name,
@@ -14,6 +16,7 @@ locals {
 
   import_inprogress_sys_topic_name = "evgt-airlock-import-in-progress-${local.topic_name_suffix}"
   import_rejected_sys_topic_name   = "evgt-airlock-import-rejected-${local.topic_name_suffix}"
+  import_blocked_sys_topic_name    = "evgt-airlock-import-blocked-${local.topic_name_suffix}"
   export_approved_sys_topic_name   = "evgt-airlock-export-approved-${local.topic_name_suffix}"
 
   scan_result_topic_name    = "evgt-airlock-scan-result-${local.topic_name_suffix}"
@@ -25,13 +28,14 @@ locals {
   scan_result_queue_name    = "airlock-scan-result"
   blob_created_topic_name   = "airlock-blob-created"
 
-  blob_created_malware_subscription_name      = "airlock-blob-created-malware-scanner"
   blob_created_al_processor_subscription_name = "airlock-blob-created-airlock-processor"
 
   step_result_eventgrid_subscription_name       = "evgs-airlock-update-status"
   status_changed_eventgrid_subscription_name    = "evgs-airlock-status-changed"
+  scan_result_eventgrid_subscription_name       = "evgs-airlock-scan-result"
   import_inprogress_eventgrid_subscription_name = "evgs-airlock-import-in-progress-blob-created"
   import_rejected_eventgrid_subscription_name   = "evgs-airlock-import-rejected-blob-created"
+  import_blocked_eventgrid_subscription_name    = "evgs-airlock-import-blocked-blob-created"
   export_approved_eventgrid_subscription_name   = "evgs-airlock-export-approved-blob-created"
 
   airlock_function_app_name = "func-airlock-processor-${var.tre_id}"

templates/core/terraform/airlock/service_bus.tf

@@ -33,11 +33,6 @@ resource "azurerm_servicebus_subscription" "airlock_processor" {
   max_delivery_count = 1
 }
 
-resource "azurerm_servicebus_subscription" "malware_scanner" {
-  name               = local.blob_created_malware_subscription_name
-  topic_id           = azurerm_servicebus_topic.blob_created.id
-  max_delivery_count = 1
-}
 
 
 

templates/core/terraform/airlock/storage_accounts.tf

@@ -141,3 +141,51 @@ resource "azurerm_private_endpoint" "stg_import_rejected_pe" {
 
   lifecycle { ignore_changes = [tags] }
 }
+
+# 'Blocked' storage account
+resource "azurerm_storage_account" "sa_import_blocked" {
+  name                            = local.import_blocked_storage_name
+  location                        = var.location
+  resource_group_name             = var.resource_group_name
+  account_tier                    = "Standard"
+  account_replication_type        = "GRS"
+  allow_nested_items_to_be_public = false
+
+  # Important! we rely on the fact that the blob craeted events are issued when the creation of the blobs are done.
+  # This is true ONLY when Hierarchical Namespace is DISABLED
+  is_hns_enabled = false
+
+  tags = merge(var.tre_core_tags, {
+    description = "airlock;import;blocked"
+  })
+
+  network_rules {
+    default_action = var.enable_local_debugging ? "Allow" : "Deny"
+    bypass         = ["AzureServices"]
+  }
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_private_endpoint" "stg_import_blocked_pe" {
+  name                = "stg-import-blocked-blob-${var.tre_id}"
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  subnet_id           = var.airlock_storage_subnet_id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group-stg-import-blocked"
+    private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-stg-import-rej-${var.tre_id}"
+    private_connection_resource_id = azurerm_storage_account.sa_import_blocked.id
+    is_manual_connection           = false
+    subresource_names              = ["Blob"]
+  }
+
+  tags = var.tre_core_tags
+
+  lifecycle { ignore_changes = [tags] }
+}

templates/workspaces/base/terraform/airlock/eventgrid_topics.tf

@@ -101,7 +101,6 @@ resource "null_resource" "wait_for_export_inprogress_blob_created" {
   depends_on = [azurerm_eventgrid_system_topic.export_inprogress_blob_created]
 }
 
-
 resource "azurerm_eventgrid_system_topic" "export_rejected_blob_created" {
   name                   = local.export_rejected_sys_topic_name
   location               = var.location
@@ -138,6 +137,56 @@ resource "azurerm_role_assignment" "servicebus_sender_export_rejected_blob_creat
   ]
 }
 
+# TEMPPORARY MITIGATION. Should be removed with https://github.com/microsoft/AzureTRE/issues/2164
+resource "null_resource" "wait_for_export_rejected_blob_created" {
+  provisioner "local-exec" {
+    command    = "bash -c \"sleep 60s\""
+    on_failure = fail
+  }
+
+  triggers = {
+    always_run = timestamp()
+  }
+
+  depends_on = [azurerm_eventgrid_system_topic.export_rejected_blob_created]
+}
+
+resource "azurerm_eventgrid_system_topic" "export_blocked_blob_created" {
+  name                   = local.export_blocked_sys_topic_name
+  location               = var.location
+  resource_group_name    = var.ws_resource_group_name
+  source_arm_resource_id = azurerm_storage_account.sa_export_blocked.id
+  topic_type             = "Microsoft.Storage.StorageAccounts"
+
+  tags = merge(
+    var.tre_workspace_tags,
+    {
+      Publishers = "airlock;export-blocked-sa"
+    }
+  )
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  depends_on = [
+    azurerm_storage_account.sa_export_blocked,
+    null_resource.wait_for_export_rejected_blob_created
+  ]
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_role_assignment" "servicebus_sender_export_blocked_blob_created" {
+  scope                = data.azurerm_servicebus_namespace.airlock_sb.id
+  role_definition_name = "Azure Service Bus Data Sender"
+  principal_id         = azurerm_eventgrid_system_topic.export_blocked_blob_created.identity.0.principal_id
+
+  depends_on = [
+    azurerm_eventgrid_system_topic.export_blocked_blob_created
+  ]
+}
+
 
 ## Subscriptions
 resource "azurerm_eventgrid_event_subscription" "import_approved_blob_created" {
@@ -184,3 +233,18 @@ resource "azurerm_eventgrid_event_subscription" "export_rejected_blob_created" {
     azurerm_eventgrid_system_topic.export_rejected_blob_created
   ]
 }
+
+resource "azurerm_eventgrid_event_subscription" "export_blocked_blob_created" {
+  name  = "export-blocked-blob-created-${var.short_workspace_id}"
+  scope = azurerm_storage_account.sa_export_blocked.id
+
+  service_bus_topic_endpoint_id = data.azurerm_servicebus_topic.blob_created.id
+
+  delivery_identity {
+    type = "SystemAssigned"
+  }
+
+  depends_on = [
+    azurerm_eventgrid_system_topic.export_blocked_blob_created
+  ]
+}

templates/workspaces/base/terraform/airlock/locals.tf

@@ -6,15 +6,18 @@ locals {
   import_approved_sys_topic_name   = "evgt-airlock-import-approved-${local.workspace_resource_name_suffix}"
   export_inprogress_sys_topic_name = "evgt-airlock-export-inprog-${local.workspace_resource_name_suffix}"
   export_rejected_sys_topic_name   = "evgt-airlock-export-rejected-${local.workspace_resource_name_suffix}"
+  export_blocked_sys_topic_name    = "evgt-airlock-export-blocked-${local.workspace_resource_name_suffix}"
 
   blob_created_topic_name = "airlock-blob-created"
 
-  # STorage AirLock APProved IMport
+  # STorage AirLock IMport APProved
   import_approved_storage_name = lower(replace("stalimapp${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", ""))
-  # STorage AirLock INTernal EXport
+  # STorage AirLock EXport INTernal
   export_internal_storage_name = lower(replace("stalexint${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", ""))
-  # STorage AirLock InProgress EXport
+  # STorage AirLock EXport InProgress
   export_inprogress_storage_name = lower(replace("stalexip${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", ""))
-  # STorage AirLock REJected EXport
+  # STorage AirLock EXport REJected
   export_rejected_storage_name = lower(replace("stalexrej${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", ""))
+  # STorage AirLock EXport BLOCKED
+  export_blocked_storage_name = lower(replace("stalexblocked${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", ""))
 }

templates/workspaces/base/terraform/airlock/storage_accounts.tf

@@ -204,6 +204,56 @@ resource "azurerm_private_endpoint" "export_rejected_pe" {
   }
 }
 
+# 'Blocked' location for export
+resource "azurerm_storage_account" "sa_export_blocked" {
+  name                            = local.export_blocked_storage_name
+  location                        = var.location
+  resource_group_name             = var.ws_resource_group_name
+  account_tier                    = "Standard"
+  account_replication_type        = "GRS"
+  allow_nested_items_to_be_public = false
+
+  # Important! we rely on the fact that the blob craeted events are issued when the creation of the blobs are done.
+  # This is true ONLY when Hierarchical Namespace is DISABLED
+  is_hns_enabled = false
+
+  network_rules {
+    default_action = var.enable_local_debugging ? "Allow" : "Deny"
+    bypass         = ["AzureServices"]
+  }
+
+  tags = merge(
+    var.tre_workspace_tags,
+    {
+      description = "airlock;export;blocked"
+    }
+  )
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+
+resource "azurerm_private_endpoint" "export_blocked_pe" {
+  name                = "pe-sa-export-blocked-blob-${var.short_workspace_id}"
+  location            = var.location
+  resource_group_name = var.ws_resource_group_name
+  subnet_id           = var.services_subnet_id
+
+  lifecycle { ignore_changes = [tags] }
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group-sa-export-blocked"
+    private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-sa-export-blocked-${var.short_workspace_id}"
+    private_connection_resource_id = azurerm_storage_account.sa_export_blocked.id
+    is_manual_connection           = false
+    subresource_names              = ["Blob"]
+  }
+}
+
 resource "azurerm_role_assignment" "sa_import_approved" {
   scope                = azurerm_storage_account.sa_import_approved.id
   role_definition_name = "Contributor"
@@ -228,3 +278,10 @@ resource "azurerm_role_assignment" "sa_export_rejected" {
   role_definition_name = "Contributor"
   principal_id         = data.azurerm_user_assigned_identity.airlock_id.principal_id
 }
+
+resource "azurerm_role_assignment" "sa_export_blocked" {
+  scope                = azurerm_storage_account.sa_export_blocked.id
+  role_definition_name = "Contributor"
+  principal_id         = data.azurerm_user_assigned_identity.airlock_id.principal_id
+}
+