Updated DBW to include virtual network integration (#2)

* Updated DBW for vnet integration

* Parameterised no public IP

data.tf

@@ -8,6 +8,24 @@ data "azurerm_log_analytics_workspace" "main" {
 }
 
 data "azurerm_storage_account" "main" {
-  name                = var.storage_account_name
+  name                = var.diagnostics_storage_account_name
-  resource_group_name = var.storage_account_resource_group_name
+  resource_group_name = var.diagnostics_storage_account_resource_group_name
 }
+
+data "azurerm_virtual_network" "main" {
+  name                = var.databricks_virtual_network_name
+  resource_group_name = var.databricks_virtual_network_resource_group_name
+}
+
+data "azurerm_subnet" "private" {
+  name                 = var.databricks_private_subnet_name
+  virtual_network_name = data.azurerm_virtual_network.main.name
+  resource_group_name  = var.databricks_virtual_network_resource_group_name
+}
+
+data "azurerm_subnet" "public" {
+  name                 = var.databricks_public_subnet_name
+  virtual_network_name = data.azurerm_virtual_network.main.name
+  resource_group_name  = var.databricks_virtual_network_resource_group_name
+}
+

examples/complete/main.tf

@@ -12,12 +12,12 @@ module "naming" {
 }
 
 resource "azurerm_resource_group" "test_group" {
-  name     = "${module.naming.resource_group.slug}-${module.naming.databricks_workspace.slug}-min-test-${local.unique_name_stub}"
+  name     = "${module.naming.resource_group.slug}-${module.naming.databricks_workspace.slug}-max-${local.unique_name_stub}"
   location = "uksouth"
 }
 
 resource "azurerm_log_analytics_workspace" "test_la" {
-  name                = "${module.naming.resource_group.slug}-${module.naming.log_analytics_workspace.slug}-min-test-${local.unique_name_stub}"
+  name                = "${module.naming.resource_group.slug}-${module.naming.log_analytics_workspace.slug}-max-${local.unique_name_stub}"
   location            = azurerm_resource_group.test_group.location
   resource_group_name = azurerm_resource_group.test_group.name
   sku                 = "PerGB2018"
@@ -31,16 +31,80 @@ resource "azurerm_storage_account" "test_sa" {
   account_replication_type = "LRS"
 }
 
-module "terraform-azurerm-databricks-workspace" {
+resource "azurerm_virtual_network" "test_vnet" {
-  source                              = "../../"
+  name                = module.naming.virtual_network.name_unique
-  resource_group_name                 = azurerm_resource_group.test_group.name
+  address_space       = ["10.0.0.0/16"]
-  log_analytics_resource_group_name   = azurerm_log_analytics_workspace.test_la.resource_group_name
+  location            = azurerm_resource_group.test_group.location
-  log_analytics_name                  = azurerm_log_analytics_workspace.test_la.name
+  resource_group_name = azurerm_resource_group.test_group.name
-  storage_account_resource_group_name = azurerm_storage_account.test_sa.resource_group_name
+}
-  storage_account_name                = azurerm_storage_account.test_sa.name
+
-  prefix                              = [local.unique_name_stub]
+resource "azurerm_subnet" "private_snet" {
-  suffix                              = [local.unique_name_stub]
+  name                 = "${module.naming.subnet.name_unique}-private"
-  databricks_workspace_sku            = "premium"
+  resource_group_name  = azurerm_resource_group.test_group.name
-  diagnostics_script_path             = "../../scripts/diagnostics.sh"
+  virtual_network_name = azurerm_virtual_network.test_vnet.name
-  module_depends_on                   = ["module.azurerm_log_analytics_workspace.test_la"]
+  address_prefixes     = ["10.0.1.0/24"]
+
+  delegation {
+    name = "databricksprivatermdelegation"
+
+    service_delegation {
+      name    = "Microsoft.Databricks/workspaces"
+    }
+  }
+}
+
+resource "azurerm_network_security_group" "private_empty_nsg" {
+  name                = "${module.naming.network_security_group.name_unique}-private"
+  location            = azurerm_resource_group.test_group.location
+  resource_group_name = azurerm_resource_group.test_group.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "private_nsg_asso" {
+  subnet_id                 = azurerm_subnet.private_snet.id
+  network_security_group_id = azurerm_network_security_group.private_empty_nsg.id
+}
+
+resource "azurerm_subnet" "public_snet" {
+  name                 = "${module.naming.subnet.name_unique}-public"
+  resource_group_name  = azurerm_resource_group.test_group.name
+  virtual_network_name = azurerm_virtual_network.test_vnet.name
+  address_prefixes     = ["10.0.2.0/24"]
+
+  delegation {
+    name = "databrickspublicdelegation"
+
+    service_delegation {
+      name    = "Microsoft.Databricks/workspaces"
+    }
+  }
+}
+
+resource "azurerm_network_security_group" "public_empty_nsg" {
+  name                = "${module.naming.network_security_group.name_unique}-public"
+  location            = azurerm_resource_group.test_group.location
+  resource_group_name = azurerm_resource_group.test_group.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "public_nsg_asso" {
+  subnet_id                 = azurerm_subnet.public_snet.id
+  network_security_group_id = azurerm_network_security_group.public_empty_nsg.id
+}
+
+module "terraform-azurerm-databricks-workspace" {
+  source                                          = "../../"
+  resource_group_name                             = azurerm_resource_group.test_group.name
+  log_analytics_resource_group_name               = azurerm_log_analytics_workspace.test_la.resource_group_name
+  log_analytics_name                              = azurerm_log_analytics_workspace.test_la.name
+  diagnostics_storage_account_resource_group_name = azurerm_storage_account.test_sa.resource_group_name
+  diagnostics_storage_account_name                = azurerm_storage_account.test_sa.name
+  databricks_virtual_network_name                 = azurerm_virtual_network.test_vnet.name
+  databricks_virtual_network_resource_group_name  = azurerm_resource_group.test_group.name
+  databricks_private_subnet_name                  = azurerm_subnet.private_snet.name
+  databricks_public_subnet_name                   = azurerm_subnet.public_snet.name
+  prefix                                          = [local.unique_name_stub]
+  suffix                                          = [local.unique_name_stub]
+  databricks_workspace_sku                        = "premium"
+  diagnostics_script_path                         = "../../scripts/diagnostics.sh"
+  no_public_ip                                    = false
+  module_depends_on                               = ["azurerm_subnet.private_snet, azurerm_subnet.public_snet"]
 }

examples/minimal/main.tf

@@ -12,12 +12,12 @@ module "naming" {
 }
 
 resource "azurerm_resource_group" "test_group" {
-  name     = "${module.naming.resource_group.slug}-${module.naming.databricks_workspace.slug}-min-test-${local.unique_name_stub}"
+  name     = "${module.naming.resource_group.slug}-${module.naming.databricks_workspace.slug}-min-${local.unique_name_stub}"
   location = "uksouth"
 }
 
 resource "azurerm_log_analytics_workspace" "test_la" {
-  name                = "${module.naming.resource_group.slug}-${module.naming.log_analytics_workspace.slug}-min-test-${local.unique_name_stub}"
+  name                = "${module.naming.resource_group.slug}-${module.naming.log_analytics_workspace.slug}-min-${local.unique_name_stub}"
   location            = azurerm_resource_group.test_group.location
   resource_group_name = azurerm_resource_group.test_group.name
   sku                 = "PerGB2018"
@@ -31,11 +31,74 @@ resource "azurerm_storage_account" "test_sa" {
   account_replication_type = "LRS"
 }
 
-module "terraform-azurerm-databricks-workspace" {
+resource "azurerm_virtual_network" "test_vnet" {
-  source                              = "../../"
+  name                = module.naming.virtual_network.name_unique
-  resource_group_name                 = azurerm_resource_group.test_group.name
+  address_space       = ["10.0.0.0/16"]
-  log_analytics_resource_group_name   = azurerm_log_analytics_workspace.test_la.resource_group_name
+  location            = azurerm_resource_group.test_group.location
-  log_analytics_name                  = azurerm_log_analytics_workspace.test_la.name
+  resource_group_name = azurerm_resource_group.test_group.name
-  storage_account_name                = azurerm_storage_account.test_sa.name
+}
-  storage_account_resource_group_name = azurerm_storage_account.test_sa.resource_group_name
+
+resource "azurerm_subnet" "private_snet" {
+  name                 = "${module.naming.subnet.name_unique}-private"
+  resource_group_name  = azurerm_resource_group.test_group.name
+  virtual_network_name = azurerm_virtual_network.test_vnet.name
+  address_prefixes     = ["10.0.1.0/24"]
+
+  delegation {
+    name = "databricksprivatermdelegation"
+
+    service_delegation {
+      name    = "Microsoft.Databricks/workspaces"
+    }
+  }
+}
+
+resource "azurerm_network_security_group" "private_empty_nsg" {
+  name                = "${module.naming.network_security_group.name_unique}-private"
+  location            = azurerm_resource_group.test_group.location
+  resource_group_name = azurerm_resource_group.test_group.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "private_nsg_asso" {
+  subnet_id                 = azurerm_subnet.private_snet.id
+  network_security_group_id = azurerm_network_security_group.private_empty_nsg.id
+}
+
+resource "azurerm_subnet" "public_snet" {
+  name                 = "${module.naming.subnet.name_unique}-public"
+  resource_group_name  = azurerm_resource_group.test_group.name
+  virtual_network_name = azurerm_virtual_network.test_vnet.name
+  address_prefixes     = ["10.0.2.0/24"]
+
+  delegation {
+    name = "databrickspublicdelegation"
+
+    service_delegation {
+      name    = "Microsoft.Databricks/workspaces"
+    }
+  }
+}
+
+resource "azurerm_network_security_group" "public_empty_nsg" {
+  name                = "${module.naming.network_security_group.name_unique}-public"
+  location            = azurerm_resource_group.test_group.location
+  resource_group_name = azurerm_resource_group.test_group.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "public_nsg_asso" {
+  subnet_id                 = azurerm_subnet.public_snet.id
+  network_security_group_id = azurerm_network_security_group.public_empty_nsg.id
+}
+
+module "terraform-azurerm-databricks-workspace" {
+  source                                          = "../../"
+  resource_group_name                             = azurerm_resource_group.test_group.name
+  log_analytics_resource_group_name               = azurerm_log_analytics_workspace.test_la.resource_group_name
+  log_analytics_name                              = azurerm_log_analytics_workspace.test_la.name
+  diagnostics_storage_account_resource_group_name = azurerm_storage_account.test_sa.resource_group_name
+  diagnostics_storage_account_name                = azurerm_storage_account.test_sa.name
+  databricks_virtual_network_name                 = azurerm_virtual_network.test_vnet.name
+  databricks_virtual_network_resource_group_name  = azurerm_resource_group.test_group.name
+  databricks_private_subnet_name                  = azurerm_subnet.private_snet.name
+  databricks_public_subnet_name                   = azurerm_subnet.public_snet.name
 }

main.tf

@@ -3,17 +3,14 @@ provider "azurerm" {
   features {}
 }
 
-resource "null_resource" "module_depends_on" {
-  triggers = {
-    value = "${length(var.module_depends_on)}"
-  }
-}
-
 locals {
-  resource_group_name        = data.azurerm_resource_group.main.name
+  resource_group                 = data.azurerm_resource_group.main
-  log_analytics_workspace_id = data.azurerm_log_analytics_workspace.main.id
+  databricks_vnet_id             = data.azurerm_virtual_network.main.id
-  storage_account_id         = data.azurerm_storage_account.main.id
+  databricks_private_snet_name   = data.azurerm_subnet.private.name
-  diagnostics_script_path    = var.diagnostics_script_path == "" ? "${path.module}/scripts/diagnostics.sh" : var.diagnostics_script_path
+  databricks_public_snet_name    = data.azurerm_subnet.public.name
+  log_analytics_workspace_id     = data.azurerm_log_analytics_workspace.main.id
+  diagnostics_storage_account_id = data.azurerm_storage_account.main.id
+  diagnostics_script_path        = var.diagnostics_script_path == "" ? "${path.module}/scripts/diagnostics.sh" : var.diagnostics_script_path
 }
 
 module "azurerm_naming" {
@@ -24,17 +21,31 @@ module "azurerm_naming" {
 
 resource "azurerm_databricks_workspace" "main" {
   name                = module.azurerm_naming.databricks_workspace.name_unique
-  resource_group_name = local.resource_group_name
+  resource_group_name = local.resource_group.name
-  location            = data.azurerm_resource_group.main.location
+  location            = local.resource_group.location
   sku                 = var.databricks_workspace_sku
+
+  custom_parameters {
+    no_public_ip        = var.no_public_ip
+    virtual_network_id  = local.databricks_vnet_id
+    public_subnet_name  = local.databricks_public_snet_name
+    private_subnet_name = local.databricks_private_snet_name
+  }
 }
 
 resource "null_resource" "main" {
   triggers = {
-    log_analytics_id   = local.log_analytics_workspace_id
+    log_analytics_id               = local.log_analytics_workspace_id
-    storage_account_id = local.storage_account_id
+    diagnostics_storage_account_id = local.diagnostics_storage_account_id
   }
   provisioner "local-exec" {
     command = "${local.diagnostics_script_path} ${local.resource_group_name} ${local.log_analytics_workspace_id} ${local.storage_account_id} ${azurerm_databricks_workspace.main.id}"
   }
 }
+
+resource "null_resource" "module_depends_on" {
+  triggers = {
+    value = "${length(var.module_depends_on)}"
+  }
+}
+

variables.tf

@@ -14,16 +14,36 @@ variable "log_analytics_resource_group_name" {
   description = "The name of a pre-existing resource group containing the desired log analytics workspace to stream logs to."
 }
 
-variable "storage_account_name" {
+variable "diagnostics_storage_account_name" {
   type        = string
   description = "The name of a pre-existing storage account to archive logs to."
 }
 
-variable "storage_account_resource_group_name" {
+variable "diagnostics_storage_account_resource_group_name" {
   type        = string
   description = "The name of a pre-existing resource group containing the desired storage account to archive logs to."
 }
 
+variable "databricks_virtual_network_name" {
+  type        = string
+  description = "The name of a pre-existing virtual network to provision the Databricks Workspace to."
+}
+
+variable "databricks_virtual_network_resource_group_name" {
+  type        = string 
+  description = "The name of the resource group in which the databricks virtual network resides in."
+}
+
+variable "databricks_private_subnet_name" {
+  type        = string
+  description = "The name of the private Databricks sub net."
+}
+
+variable "databricks_public_subnet_name" {
+  type        = string
+  description = "The name of the public Databricks sub net."
+}
+
 # Optional variables
 variable "prefix" {
   type        = list(string)
@@ -49,6 +69,13 @@ variable "diagnostics_script_path" {
   default     = ""
 }
 
+variable "no_public_ip" {
+  type        = bool
+  description = "A boolean determining whether or not to initialise the Azure Databricks Workspace with a public IP address."
+  #NOTE: Default to false here as not every Azure Subscription is by default capable of instantiating Databricks Workspaces with no public IP address. 
+  default     = false
+}
+
 variable "module_depends_on" {
   default = [""]
 }