Adding Azure databricks workspace service (#1857)

* Azure Databricks TRE workspace service Co-authored-by: Guy Bertental <guybartal@gmail.com> Co-authored-by: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Co-authored-by: Ross Smith <ross-p-smith@users.noreply.github.com> Co-authored-by: Marcus Robinson <marrobi@microsoft.com>
2023-01-31 20:42:14 +11:00 · 2023-01-31 20:42:14 +11:00 · 35b486b100
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -24,6 +24,7 @@
  :warning: Any custom rules you have added manually will be **lost** and you'll need to add it back after the upgrade has been completed.

 FEATURES:
+* Add Azure Databricks as workspace service [#1857](https://github.com/microsoft/AzureTRE/pull/1857)

 ENHANCEMENTS:
 * Add support for referencing IP Groups from the Core Resource Group in firewall rules created via the pipeline [#3089](https://github.com/microsoft/AzureTRE/pull/3089)
--- a/core/terraform/locals.tf
+++ b/core/terraform/locals.tf
@ -30,5 +30,6 @@ locals {
    "privatelink.postgres.database.azure.com",
    "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com",
    "privatelink.mysql.database.azure.com",
+    "privatelink.azuredatabricks.net"
  ])
 }
--- a/core/terraform/network/locals.tf
+++ b/core/terraform/network/locals.tf
@ -29,6 +29,6 @@ locals {

  private_dns_zone_names = toset([
    "privatelink.queue.core.windows.net",
-    "privatelink.table.core.windows.net",
+    "privatelink.table.core.windows.net"
  ])
 }
--- a/core/version.txt
+++ b/core/version.txt
@ -1 +1 @@
-__version__ = "0.7.1"
+__version__ = "0.7.2"
--- a/docs/assets/databricks_workspace_service.png
+++ b/docs/assets/databricks_workspace_service.png
--- a/docs/tre-templates/workspace-services/databricks.md
+++ b/docs/tre-templates/workspace-services/databricks.md
@ -0,0 +1,17 @@
+# Azure Databricks workspace service bundle
+
+See: [https://azure.microsoft.com/en-us/products/databricks/](https://azure.microsoft.com/en-us/products/databricks/)
+
+This service installs the following resources into an existing virtual network within the workspace:
+
+![Azure Databricks workspace service](../../assets/databricks_workspace_service.png)
+
+
+## Properties
+
+- `is_exposed_externally` - If `True`, the Azure Databricks workspace is accessible from outside of the worksapce virtual network. If `False` use a Guacamole VM and the `internal_connection_uri` to access Databricks workspace.
+
+
+## Prerequisites
+
+- [A base workspace bundle installed](../workspaces/base.md)
--- a/mkdocs.yml
+++ b/mkdocs.yml
@ -82,11 +82,11 @@ nav:
          - Set up of a Virtual Machine: using-tre/tre-for-research/using-vms.md
          - Importing/exporting data with Airlock: using-tre/tre-for-research/importing-exporting-data-airlock.md
          - Reviewing Airlock Requests: using-tre/tre-for-research/review-airlock-request.md
-      # - Workspaces:
-      #    - using-tre/wks/index.md # Documentation describing what a workspace is
-      #    - Using Workspaces: using-tre/wks/using-wks.md # Interacting with workspaces (via the UI)
-      #    - The Workspace Owner: using-tre/wks/wks-owner.md # Workspace Owners. The concept, and tasks
-      # - FAQ: using-tre/faq.md # FAQ section (to allow easy contribution)
+          # - Workspaces:
+          #    - using-tre/wks/index.md # Documentation describing what a workspace is
+          #    - Using Workspaces: using-tre/wks/using-wks.md # Interacting with workspaces (via the UI)
+          #    - The Workspace Owner: using-tre/wks/wks-owner.md # Workspace Owners. The concept, and tasks
+          # - FAQ: using-tre/faq.md # FAQ section (to allow easy contribution)

  - Templates and Services: # Docs to highlight and illustrate workspaces, workspace services etc
      - Workspaces:
@ -100,6 +100,7 @@ nav:
          - InnerEye: tre-templates/workspace-services/inner-eye.md
          - MLFlow: tre-templates/workspace-services/mlflow.md
          - Health Services: tre-templates/workspace-services/health_services.md
+          - Azure Databricks: tre-templates/workspace-services/databricks.md
      - Shared Services:
          - Gitea (Source Mirror): tre-templates/shared-services/gitea.md
          - Nexus (Package Mirror): tre-templates/shared-services/nexus.md
@ -129,6 +130,7 @@ nav:
          - Registering Templates: tre-admins/registering-templates.md
          - Install Resources via API:
              - Install Base Workspace: tre-admins/setup-instructions/installing-base-workspace.md
+              # yamllint disable-line rule:line-length
              - Install Workspace Service and User Resource: tre-admins/setup-instructions/installing-workspace-service-and-user-resource.md
          - Upgrading AzureTRE Version: tre-admins/upgrading-tre.md
          - Upgrading Resources Version: tre-admins/upgrading-resources.md
--- a/templates/workspace_services/databricks/.dockerignore
+++ b/templates/workspace_services/databricks/.dockerignore
@ -0,0 +1,8 @@
+# Local .terraform directories
+**/.terraform/*
+
+# TF backend files
+**/*_backend.tf
+Dockerfile.tmpl
+terraform/deploy.sh
+terraform/destroy.sh
--- a/templates/workspace_services/databricks/.env.sample
+++ b/templates/workspace_services/databricks/.env.sample
@ -0,0 +1,5 @@
+ID=__CHANGE_ME__
+WORKSPACE_ID=__CHANGE_ME__
+AZURE_LOCATION=__CHANGE_ME__
+HOST_SUBNET_ADDRESS_PREFIX=__CHANGE_ME__
+CONTAINER_SUBNET_ADDRESS_PREFIX=__CHANGE_ME__
--- a/templates/workspace_services/databricks/Dockerfile.tmpl
+++ b/templates/workspace_services/databricks/Dockerfile.tmpl
@ -0,0 +1,18 @@
+# syntax=docker/dockerfile-upstream:1.4.0
+FROM debian:bullseye-slim
+
+# PORTER_INIT
+
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
+# Install git - required for https://registry.terraform.io/modules/claranet/regions/azurerm
+RUN apt-get update && apt-get install --no-install-recommends -y git \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# PORTER_MIXINS
+
+RUN apt-get remove -y git
+
+# Use the BUNDLE_DIR build argument to copy files into the bundle
+COPY --link . ${BUNDLE_DIR}/
--- a/templates/workspace_services/databricks/README.md
+++ b/templates/workspace_services/databricks/README.md
@ -0,0 +1,47 @@
+# Contents
+
+## porter.yaml
+
+This is the porter manifest. See <https://porter.sh/author-bundles/> for
+details on every field and how to configure your bundle. This is a required
+file.
+
+## helpers.sh
+
+This is a bash script where you can place helper functions that you can call
+from your porter.yaml file.
+
+## README.md
+
+This explains the files created by `porter create`. It is not used by porter and
+can be deleted.
+
+## Dockerfile.tmpl
+
+This is a template Dockerfile for the bundle's invocation image. You can
+customize it to use different base images, install tools and copy configuration
+files. Porter will use it as a template and append lines to it for the mixin and to set
+the CMD appropriately for the CNAB specification. You can delete this file if you don't
+need it.
+
+Add the following line to **porter.yaml** to enable the Dockerfile template:
+
+```yaml
+dockerfile: Dockerfile.tmpl
+```
+
+By default, the Dockerfile template is disabled and Porter automatically copies
+all of the files in the current directory into the bundle's invocation image. When
+you use a custom Dockerfile template, you must manually copy files into the bundle
+using COPY statements in the Dockerfile template.
+
+## .gitignore
+
+This is a default file that we provide to help remind you which files are
+generated by Porter, and shouldn't be committed to source control. You can
+delete it if you don't need it.
+
+## .dockerignore
+
+This is a default file that controls which files are copied into the bundle's
+invocation image by default. You can delete it if you don't need it.
--- a/templates/workspace_services/databricks/parameters.json
+++ b/templates/workspace_services/databricks/parameters.json
@ -0,0 +1,56 @@
+{
+  "schemaType": "ParameterSet",
+  "schemaVersion": "1.0.1",
+  "namespace": "",
+  "name": "tre-service-databricks",
+  "parameters": [
+    {
+      "name": "id",
+      "source": {
+        "env": "ID"
+      }
+    },
+    {
+      "name": "tre_id",
+      "source": {
+        "env": "TRE_ID"
+      }
+    },
+    {
+      "name": "workspace_id",
+      "source": {
+        "env": "WORKSPACE_ID"
+      }
+    },
+    {
+      "name": "address_space",
+      "source": {
+        "env": "ADDRESS_SPACE"
+      }
+    },
+    {
+      "name": "is_exposed_externally",
+      "source": {
+        "env": "IS_EXPOSED_EXTERNALLY"
+      }
+    },
+    {
+      "name": "tfstate_container_name",
+      "source": {
+        "env": "TERRAFORM_STATE_CONTAINER_NAME"
+      }
+    },
+    {
+      "name": "tfstate_resource_group_name",
+      "source": {
+        "env": "MGMT_RESOURCE_GROUP_NAME"
+      }
+    },
+    {
+      "name": "tfstate_storage_account_name",
+      "source": {
+        "env": "MGMT_STORAGE_ACCOUNT_NAME"
+      }
+    }
+  ]
+}
--- a/templates/workspace_services/databricks/porter.yaml
+++ b/templates/workspace_services/databricks/porter.yaml
@ -0,0 +1,176 @@
+---
+schemaVersion: 1.0.0
+name: tre-service-databricks
+version: 0.1.71
+description: "An Azure TRE service for Azure Databricks."
+registry: azuretre
+dockerfile: Dockerfile.tmpl
+
+credentials:
+
+  - name: azure_tenant_id
+    env: ARM_TENANT_ID
+  - name: azure_subscription_id
+    env: ARM_SUBSCRIPTION_ID
+  - name: azure_client_id
+    env: ARM_CLIENT_ID
+  - name: azure_client_secret
+    env: ARM_CLIENT_SECRET
+
+parameters:
+  - name: workspace_id
+    type: string
+  - name: tre_id
+    type: string
+  - name: id
+    type: string
+    description: "Resource ID"
+  - name: address_space
+    type: string
+  - name: is_exposed_externally
+    type: boolean
+  - name: tfstate_resource_group_name
+    type: string
+    description: "Resource group containing the Terraform state storage account"
+  - name: tfstate_storage_account_name
+    type: string
+    description: "The name of the Terraform state storage account"
+  - name: tfstate_container_name
+    env: tfstate_container_name
+    type: string
+    default: "tfstate"
+    description: "The name of the Terraform state storage container"
+  - name: arm_use_msi
+    env: ARM_USE_MSI
+    type: boolean
+    default: false
+
+outputs:
+  - name: databricks_workspace_name
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: connection_uri
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: internal_connection_uri
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: databricks_storage_account_name
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: dbfs_blob_storage_domain
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: metastore_addresses
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: event_hub_endpoint_addresses
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: log_blob_storage_domains
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: artifact_blob_storage_domains
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: workspace_address_spaces
+    type: string
+    applyTo:
+      - install
+      - upgrade
+  - name: databricks_address_prefixes
+    type: string
+    applyTo:
+      - install
+      - upgrade
+
+mixins:
+  - terraform:
+      clientVersion: 1.3.6
+
+install:
+  - terraform:
+      description: "Deploy Databricks Service"
+      vars:
+        tre_resource_id: ${ bundle.parameters.id }
+        tre_id: ${ bundle.parameters.tre_id }
+        workspace_id: ${ bundle.parameters.workspace_id }
+        address_space: ${ bundle.parameters.address_space }
+        is_exposed_externally: ${ bundle.parameters.is_exposed_externally }
+      backendConfig:
+        resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
+        storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
+        container_name: ${ bundle.parameters.tfstate_container_name }
+        key: ${ bundle.name }-${ bundle.parameters.id }
+      outputs:
+        - name: databricks_workspace_name
+        - name: connection_uri
+        - name: internal_connection_uri
+        - name: databricks_storage_account_name
+        - name: dbfs_blob_storage_domain
+        - name: metastore_addresses
+        - name: event_hub_endpoint_addresses
+        - name: log_blob_storage_domains
+        - name: artifact_blob_storage_domains
+        - name: workspace_address_spaces
+        - name: databricks_address_prefixes
+
+upgrade:
+  - terraform:
+      description: "Upgrade Databricks Service"
+      vars:
+        tre_resource_id: ${ bundle.parameters.id }
+        tre_id: ${ bundle.parameters.tre_id }
+        workspace_id: ${ bundle.parameters.workspace_id }
+        address_space: ${ bundle.parameters.address_space }
+        is_exposed_externally: ${ bundle.parameters.is_exposed_externally }
+      backendConfig:
+        resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
+        storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
+        container_name: ${ bundle.parameters.tfstate_container_name }
+        key: ${ bundle.name }-${ bundle.parameters.id }
+      outputs:
+        - name: databricks_workspace_name
+        - name: connection_uri
+        - name: internal_connection_uri
+        - name: databricks_storage_account_name
+        - name: dbfs_blob_storage_domain
+        - name: metastore_addresses
+        - name: event_hub_endpoint_addresses
+        - name: log_blob_storage_domains
+        - name: artifact_blob_storage_domains
+        - name: workspace_address_spaces
+        - name: databricks_address_prefixes
+
+uninstall:
+  - terraform:
+      description: "Uninstall Azure Databricks Service"
+      vars:
+        tre_resource_id: ${ bundle.parameters.id }
+        tre_id: ${ bundle.parameters.tre_id }
+        workspace_id: ${ bundle.parameters.workspace_id }
+        address_space: ${ bundle.parameters.address_space }
+        is_exposed_externally: ${ bundle.parameters.is_exposed_externally }
+      backendConfig:
+        resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
+        storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
+        container_name: ${ bundle.parameters.tfstate_container_name }
+        key: ${ bundle.name }-${ bundle.parameters.id }
--- a/templates/workspace_services/databricks/template_schema.json
+++ b/templates/workspace_services/databricks/template_schema.json
@ -0,0 +1,365 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "$id": "https://github.com/microsoft/AzureTRE/templates/workspace_services/databricks/template_schema.json",
+  "type": "object",
+  "title": "Azure Databricks",
+  "description": "Azure Databricks",
+  "required": [],
+  "properties": {
+    "display_name": {
+      "type": "string",
+      "title": "Name for the workspace service",
+      "description": "The name of the workspace service to be displayed to users",
+      "default": "Azure Databricks",
+      "updateable": true
+    },
+    "description": {
+      "type": "string",
+      "title": "Description of the workspace service",
+      "description": "Description of the workspace service",
+      "default": "Azure Databricks is a fast, easy, and collaborative Apache Spark-based big data analytics service designed for data science and data engineering.",
+      "updateable": true
+    },
+    "overview": {
+      "type": "string",
+      "title": "Workspace Service Overview",
+      "description": "Long form description of the workspace service, in markdown syntax",
+      "default": "The Azure Databricks Lakehouse Platform provides a unified set of tools for building, deploying, sharing, and maintaining enterprise-grade data solutions at scale.\nAzure Databricks integrates with cloud storage and security in your cloud account, and manages and deploys cloud infrastructure on your behalf.\n[Azure Databricks documentation](https://learn.microsoft.com/en-us/azure/databricks/introduction/)",
+      "updateable": true
+    },
+    "is_exposed_externally": {
+      "$id": "#/properties/is_exposed_externally",
+      "type": "boolean",
+      "title": "Expose externally",
+      "description": "Is the Databricks workspace accessible from outside of the workspace network",
+      "default": false
+    },
+    "address_space": {
+      "$id": "#/properties/address_space",
+      "type": "string",
+      "title": "Address space",
+      "description": "The address space of the databricks subnets"
+    }
+  },
+  "uiSchema": {
+    "address_space": {
+      "classNames": "tre-hidden"
+    }
+  },
+  "pipeline": {
+    "install": [
+      {
+        "stepId": "12ba0dad-ea6c-4d0d-9255-daa6212f5ffa",
+        "stepTitle": "Upgrade to ensure aware of address space",
+        "resourceType": "workspace",
+        "resourceAction": "upgrade",
+        "properties": []
+      },
+      {
+        "stepId": "main"
+      },
+      {
+        "stepId": "260421b3-7308-491f-b531-e007cdc0aa46",
+        "stepTitle": "Add firewall rules for databricks",
+        "resourceTemplateName": "tre-shared-service-firewall",
+        "resourceType": "shared-service",
+        "resourceAction": "upgrade",
+        "properties": [
+          {
+            "name": "network_rule_collections",
+            "type": "array",
+            "arraySubstitutionAction": "replace",
+            "arrayMatchField": "name",
+            "value": {
+              "name": "nrc_svc_{{ resource.id }}_databricks",
+              "action": "Allow",
+              "rules": [
+                {
+                  "name": "databricks",
+                  "description": "Communication with Azure Databricks dependancies.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "destination_addresses": [ "AzureDatabricks"],
+                  "destination_ports": [
+                    "443"
+                  ],
+                  "protocols": [
+                    "TCP"
+                  ]
+                },
+                {
+                  "name": "databricks-sql-metastore",
+                  "description": "Stores metadata for databases and child objects in a Azure Databricks workspace.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "destination_addresses": "{{ resource.properties.metastore_addresses }}",
+                  "destination_ports": [
+                    "3306"
+                  ],
+                  "protocols": [
+                    "TCP"
+                  ]
+                },
+                {
+                  "name": "databricks-observability-eventhub",
+                  "description": "Transit for Azure Databricks on-cluster service specific telemetry.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "destination_addresses": "{{ resource.properties.event_hub_endpoint_addresses }}",
+                  "destination_ports": [
+                    "9093"
+                  ],
+                  "protocols": [
+                    "TCP"
+                  ]
+                },
+                {
+                  "name": "AzureAD",
+                  "description": "AAD access",
+                  "source_addresses": "{{ resource.properties.workspace_address_spaces }}",
+                  "destination_addresses": ["AzureActiveDirectory"],
+                  "destination_ports": ["*"],
+                  "protocols": ["TCP"]
+                }
+              ]
+            }
+          },
+          {
+            "name": "rule_collections",
+            "type": "array",
+            "arraySubstitutionAction": "replace",
+            "arrayMatchField": "name",
+            "value": {
+              "name": "arc_svc_{{ resource.id }}_databricks",
+              "action": "Allow",
+              "rules": [
+                {
+                  "name": "databricks-spark-log-blob-storage",
+                  "description": "To store Azure Databricks audit and cluster logs (anonymized / masked) for support and troubleshooting.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "target_fqdns": "{{ resource.properties.log_blob_storage_domains }}",
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                },
+                {
+                  "name": "databricks-artifact-blob-storage",
+                  "description": "Stores Databricks Runtime images to be deployed on cluster nodes.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "target_fqdns": "{{ resource.properties.artifact_blob_storage_domains }}",
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                },
+                {
+                  "name": "databricks-dbfs",
+                  "description": "Azure Databricks workspace root storage.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "target_fqdns": [
+                    "{{ resource.properties.dbfs_blob_storage_domain }}"
+                  ],
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                },
+                {
+                  "name": "AAD CDN",
+                  "description": "AAD CDN",
+                  "source_addresses": "{{ resource.properties.workspace_address_spaces }}",
+                  "target_fqdns": [
+                    "aadcdn.msftauth.net"
+                  ],
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                }
+              ]
+            }
+          }
+        ]
+      }
+    ],
+    "upgrade": [
+      {
+        "stepId": "12baaaad-ea6c-4d0d-9255-d316212f5ffa",
+        "stepTitle": "Upgrade to ensure aware of address space",
+        "resourceType": "workspace",
+        "resourceAction": "upgrade",
+        "properties": []
+      },
+      {
+        "stepId": "main"
+      },
+      {
+        "stepId": "260421b3-7308-491f-b531-e007cdc0aa46",
+        "stepTitle": "Add firewall rules for databricks",
+        "resourceTemplateName": "tre-shared-service-firewall",
+        "resourceType": "shared-service",
+        "resourceAction": "upgrade",
+        "properties": [
+          {
+            "name": "network_rule_collections",
+            "type": "array",
+            "arraySubstitutionAction": "replace",
+            "arrayMatchField": "name",
+            "value": {
+              "name": "nrc_svc_{{ resource.id }}_databricks",
+              "action": "Allow",
+              "rules": [
+                {
+                  "name": "databricks",
+                  "description": "Communication with Azure Databricks dependancies.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "destination_addresses": [ "AzureDatabricks"],
+                  "destination_ports": [
+                    "443"
+                  ],
+                  "protocols": [
+                    "TCP"
+                  ]
+                },
+                {
+                  "name": "databricks-sql-metastore",
+                  "description": "Stores metadata for databases and child objects in a Azure Databricks workspace.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "destination_addresses": "{{ resource.properties.metastore_addresses }}",
+                  "destination_ports": [
+                    "3306"
+                  ],
+                  "protocols": [
+                    "TCP"
+                  ]
+                },
+                {
+                  "name": "databricks-observability-eventhub",
+                  "description": "Transit for Azure Databricks on-cluster service specific telemetry.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "destination_addresses": "{{ resource.properties.event_hub_endpoint_addresses }}",
+                  "destination_ports": [
+                    "9093"
+                  ],
+                  "protocols": [
+                    "TCP"
+                  ]
+                },
+                {
+                  "name": "AzureAD",
+                  "description": "AAD access",
+                  "source_addresses": "{{ resource.properties.workspace_address_spaces }}",
+                  "destination_addresses": ["AzureActiveDirectory"],
+                  "destination_ports": ["*"],
+                  "protocols": ["TCP"]
+                }
+              ]
+            }
+          },
+          {
+            "name": "rule_collections",
+            "type": "array",
+            "arraySubstitutionAction": "replace",
+            "arrayMatchField": "name",
+            "value": {
+              "name": "arc_svc_{{ resource.id }}_databricks",
+              "action": "Allow",
+              "rules": [
+                {
+                  "name": "databricks-spark-log-blob-storage",
+                  "description": "To store Azure Databricks audit and cluster logs (anonymized / masked) for support and troubleshooting.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "target_fqdns": "{{ resource.properties.log_blob_storage_domains }}",
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                },
+                {
+                  "name": "databricks-artifact-blob-storage",
+                  "description": "Stores Databricks Runtime images to be deployed on cluster nodes.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "target_fqdns": "{{ resource.properties.artifact_blob_storage_domains }}",
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                },
+                {
+                  "name": "databricks-dbfs",
+                  "description": "Azure Databricks workspace root storage.",
+                  "source_addresses": "{{ resource.properties.databricks_address_prefixes }}",
+                  "target_fqdns": [
+                    "{{ resource.properties.dbfs_blob_storage_domain }}"
+                  ],
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                },
+                {
+                  "name": "AAD CDN",
+                  "description": "AAD CDN",
+                  "source_addresses": "{{ resource.properties.workspace_address_spaces }}",
+                  "target_fqdns": [
+                    "aadcdn.msftauth.net"
+                  ],
+                  "protocols": [
+                    {
+                      "port": "443",
+                      "type": "Https"
+                    }
+                  ]
+                }
+              ]
+            }
+          }
+        ]
+      }
+    ],
+    "uninstall": [
+      {
+        "stepId": "260421b3-7308-491f-b531-e007cdc0aa46",
+        "stepTitle": "Remove network firewall rules for databricks",
+        "resourceTemplateName": "tre-shared-service-firewall",
+        "resourceType": "shared-service",
+        "resourceAction": "upgrade",
+        "properties": [
+          {
+            "name": "network_rule_collections",
+            "type": "array",
+            "arraySubstitutionAction": "remove",
+            "arrayMatchField": "name",
+            "value": {
+              "name": "nrc_svc_{{ resource.id }}_databricks"
+            }
+          },
+          {
+            "name": "rule_collections",
+            "type": "array",
+            "arraySubstitutionAction": "remove",
+            "arrayMatchField": "name",
+            "value": {
+              "name": "arc_svc_{{ resource.id }}_databricks"
+            }
+          }
+        ]
+      },
+      {
+        "stepId": "main"
+      }
+    ]
+  }
+}
--- a/templates/workspace_services/databricks/terraform/.terraform.lock.hcl
+++ b/templates/workspace_services/databricks/terraform/.terraform.lock.hcl
@ -0,0 +1,80 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/azure/azapi" {
+  version     = "1.1.0"
+  constraints = "1.1.0"
+  hashes = [
+    "h1:IR+AHCwfjl1c0baWwfOwZ6QZtHj41H2syTgHkJtAr/M=",
+    "zh:2a25df6325a49f9e821f0b02c7da86167fc19a3bac647cd1edf231300f29d077",
+    "zh:2b443a836a39724663fe455d4deee408ff3a2d9a8b86f8408aa7db2e8aa743f8",
+    "zh:364ed09ddfc50d9bed8d930f7de489cb654a9908feb139413a097823a50075fd",
+    "zh:523bc005f56ae785867d230d55c29f59db4b599dbc6c38b4d03ea55a79458916",
+    "zh:60ded375fdb305b60bcb4d9e596dbb222cab166bad1b4958199b05a72aaeacfd",
+    "zh:61e69c58642fead6814e511c872b7c0a6478ec6af4ab758b4512607d910ac078",
+    "zh:823b2154ae2262dabcbd11aac992e3cc29eae0f7baa96bee1e3e2fe1ece8730b",
+    "zh:870ea9cc24807ef5142e4cad0281dac7173f7b6bf818a79762b6c690d12d4c4b",
+    "zh:9094ae76ed66cb328a4f35bd18b9140fb6fc6859c2e46431ec73c018bcb58d96",
+    "zh:d89149cfd01cb70012459536b4d36490b58e43312440562e5910bd5160537858",
+    "zh:dba7ec06171ca062fc423ba5b4776a5600444e45e57f4d1cb043bdc3eee538b7",
+    "zh:ff5bd6883d9ac8334e043434246357a55107411e9a962856c1d17e47ee15ac37",
+  ]
+}
+
+provider "registry.terraform.io/databricks/databricks" {
+  version     = "1.5.0"
+  constraints = "1.5.0"
+  hashes = [
+    "h1:UJe5L/BteOU7M5ewRLzuUjiewYFLF695eLp3hMKVR6M=",
+    "zh:0fa9ca13d977a8dcb46254f07c9be731891468f5b423f09cb51da97eaace8e2b",
+    "zh:3a648e4f8ece8aab05acfc7759b4e4cd153ecd29b3ed0e00d7f1a3a19911f7d8",
+    "zh:3b052b98b5e22ae4e81e4b667ae5cee9a68bb1750d22546ae9eff16c8d6a294a",
+    "zh:4320b165218cb39f0ad313d483bba20d0de9e48db0c1467fd0e3a0afb2c02012",
+    "zh:588c9fdbf35ca9c430cafb5dbd90f34a165744e3514212d0f2c07a3387d8b339",
+    "zh:b50f8eb38b556ddfa24a76b4113e8a84b778a9a0bb4b4ba5fdc3edca59198d2a",
+    "zh:ca5186443ac672f5566d9c9b5727f55124a8642dd3949e973790b9195e6b306a",
+    "zh:db817409b94c34c9b9b5e109751eff7fbca90d08b407a099630c8ec79b6c6d4b",
+    "zh:edf04424c68db603bf2473e2f14f3e3ad217feb84fc2c7debb6641d15886f8e3",
+    "zh:ef374f84c41fe529bff1ec3274eb7fe5dd8184c5e71f3e6d99a6adaff6eab82e",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/azurerm" {
+  version     = "3.40.0"
+  constraints = "3.40.0"
+  hashes = [
+    "h1:/Jbhw/zNAsDYDoASaG6w+0KZyay9BkUVOpR8b7m0CsA=",
+    "zh:00fa6dc05bf2643c6a3c741edb7d88263698086835a8a613f1d7bd76d1b918fd",
+    "zh:0da9b788e773272a7aa9d59bd9e3d5842edd4acc8c3895bea469e66dc14205a0",
+    "zh:25a8c39d1f042fc7c83ba9dd745c3569ea9e577fadb57563a575fb115ac2b9f1",
+    "zh:4423666dbeae8bc22c6e8898ffbb88745681dc27668ca9104b665dd7f3d7292c",
+    "zh:78c07308e7407b558d15737a98fb5eaf15529d297fc3798de6a7d61e0466e2e3",
+    "zh:894aca7e6f4f331ee8eb51957a180dc03d399d2b1727e0d7842e9b3f022a8c6a",
+    "zh:bb0e620c2161b4c4892a6f50b1c4c69ed70f66bb5e92543a03d79d0e4b1d9441",
+    "zh:c7d8e6a791159ca63b30908c9efe72ab65f60d64b30f0c1eb5a64972f4994844",
+    "zh:d04c11bfd346c1ac34d16bbdca70b23b006e822f6beb236b85375e8343888eb4",
+    "zh:f4edea9660327c7c70a823d786fd1b1c1b186c8759770447f63da72f23e1a73c",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:f986e268949cf445ff53a66af48a87c6f6dba5964e8a5b1dc0ea02afabdd71f7",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/dns" {
+  version     = "3.2.3"
+  constraints = "3.2.3"
+  hashes = [
+    "h1:ODcR+vWOhCAJ2iCChZMVdRglNCx07VNr67OPLRPZyDY=",
+    "zh:03a304f4b76ac6c8bebffddcdf555bf77578a7f638948a681589def32e140cb8",
+    "zh:08c7d2498b747054e9c9df7838bfa4e4a6b5d63e2d29f0457247e384f792d56c",
+    "zh:20adf489819ba51ba9d9d15da2dbe1fecb92491b3d0dd80096873e5e84d8b4bd",
+    "zh:2959ff209d2578456ca490672b82864d483b9e9db9efc8e4ffada06e23017609",
+    "zh:3ecd0b22db79550fb1108ff7bd00c4066825e8c23bb64e3cc8d9b8102e8caa45",
+    "zh:6e53a9232245b4be52b56b078f15f270b89afe6abb9c9b8baab4a282fe0cf9f8",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:80437bdfa08eb90f70105b52cb06799a8f7967313654b43d28d7f654fcd4edc1",
+    "zh:816ddaca0ecc29e287376e5b0b8b0729ee13f23a9d74bfad5b14b7983e1a1775",
+    "zh:82d8ac7ad00c1a71d0a7c1aca03bb59a6b51128f895242df80b1f3d016c3c51a",
+    "zh:ec9243b8bd80693a6eeeea5d4f7f4e6f57bd44ae796d6d5b1a91790e359f8a61",
+    "zh:fd821adbfb03a2c9eac111ff27a32b3a5523b18f80333008de85482d3bbea645",
+  ]
+}
--- a/templates/workspace_services/databricks/terraform/data.tf
+++ b/templates/workspace_services/databricks/terraform/data.tf
@ -0,0 +1,30 @@
+data "azurerm_resource_group" "ws" {
+  name = local.resource_group_name
+}
+
+data "azurerm_virtual_network" "ws" {
+  name                = local.virtual_network_name
+  resource_group_name = data.azurerm_resource_group.ws.name
+}
+
+data "azurerm_firewall" "firewall" {
+  name                = local.firewall_name
+  resource_group_name = local.core_resource_group_name
+}
+data "azurerm_subnet" "services" {
+  name                 = "ServicesSubnet"
+  virtual_network_name = data.azurerm_virtual_network.ws.name
+  resource_group_name  = data.azurerm_virtual_network.ws.resource_group_name
+}
+
+data "azurerm_private_dns_zone" "databricks" {
+  name                = "privatelink.azuredatabricks.net"
+  resource_group_name = local.core_resource_group_name
+}
+
+data "azurerm_subscription" "current" {}
+
+data "azurerm_private_dns_zone" "blobcore" {
+  name                = "privatelink.blob.core.windows.net"
+  resource_group_name = local.core_resource_group_name
+}
--- a/templates/workspace_services/databricks/terraform/databricks-udr.json
+++ b/templates/workspace_services/databricks/terraform/databricks-udr.json
--- a/templates/workspace_services/databricks/terraform/deploy.sh
+++ b/templates/workspace_services/databricks/terraform/deploy.sh
@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Uncomment this line to see each command for debugging (careful: this will show secrets!)
+#set -o xtrace
+
+export TF_LOG="TRACE"
+export TF_LOG_PATH="/home/adminuser/tf.log"
+
+# shellcheck disable=SC2154
+terraform init -input=false -backend=true -reconfigure \
+    -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \
+    -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \
+    -backend-config="container_name=$TF_VAR_terraform_state_container_name" \
+    -backend-config="key=tre-workspace-service-gitea-${TF_VAR_id}"
+
+terraform plan
+
+terraform apply -auto-approve
--- a/templates/workspace_services/databricks/terraform/locals.tf
+++ b/templates/workspace_services/databricks/terraform/locals.tf
@ -0,0 +1,28 @@
+locals {
+  databricks_subnets             = cidrsubnets(var.address_space, 1, 1)
+  container_subnet_address_space = local.databricks_subnets[0] # .0 - .127
+  host_subnet_address_space      = local.databricks_subnets[1] # .128 - .254
+  short_service_id               = substr(var.tre_resource_id, -4, -1)
+  short_workspace_id             = substr(var.workspace_id, -4, -1)
+  workspace_resource_name_suffix = "${var.tre_id}-ws-${local.short_workspace_id}"
+  service_resource_name_suffix   = "${var.tre_id}-ws-${local.short_workspace_id}-svc-${local.short_service_id}"
+  resource_group_name            = "rg-${var.tre_id}-ws-${local.short_workspace_id}"
+  virtual_network_name           = "vnet-${local.workspace_resource_name_suffix}"
+  core_resource_group_name       = "rg-${var.tre_id}"
+  firewall_name                  = "fw-${var.tre_id}"
+  databricks_workspace_name      = "adb-${local.service_resource_name_suffix}"
+  managed_resource_group_name    = "rg-${local.service_resource_name_suffix}"
+  host_subnet_name               = "adb-host-subnet-${local.service_resource_name_suffix}"
+  container_subnet_name          = "adb-container-subnet-${local.service_resource_name_suffix}"
+  network_security_group_name    = "nsg-${local.service_resource_name_suffix}"
+  route_table_name               = "rt-${local.service_resource_name_suffix}"
+  # databricks-udr.json was build according to this page https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/udr
+  map_location_url_config = jsondecode(file("${path.module}/databricks-udr.json"))
+  storage_name            = lower(replace("stgdbfs${substr(local.service_resource_name_suffix, -8, -1)}", "-", ""))
+
+  tre_workspace_service_tags = {
+    tre_id                   = var.tre_id
+    tre_workspace_id         = var.workspace_id
+    tre_workspace_service_id = var.tre_resource_id
+  }
+}
--- a/templates/workspace_services/databricks/terraform/main.tf
+++ b/templates/workspace_services/databricks/terraform/main.tf
@ -0,0 +1,28 @@
+resource "azurerm_databricks_workspace" "databricks" {
+  name                                  = local.databricks_workspace_name
+  resource_group_name                   = data.azurerm_resource_group.ws.name
+  location                              = data.azurerm_resource_group.ws.location
+  sku                                   = "premium"
+  managed_resource_group_name           = local.managed_resource_group_name
+  infrastructure_encryption_enabled     = true
+  public_network_access_enabled         = var.is_exposed_externally
+  network_security_group_rules_required = var.is_exposed_externally ? "AllRules" : "NoAzureDatabricksRules"
+  tags                                  = local.tre_workspace_service_tags
+
+  lifecycle { ignore_changes = [tags] }
+
+  custom_parameters {
+    no_public_ip                                         = true
+    public_subnet_name                                   = azurerm_subnet.host.name
+    private_subnet_name                                  = azurerm_subnet.container.name
+    virtual_network_id                                   = data.azurerm_virtual_network.ws.id
+    public_subnet_network_security_group_association_id  = azurerm_subnet_network_security_group_association.host.id
+    private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.container.id
+    storage_account_name                                 = local.storage_name
+  }
+
+  depends_on = [
+    azurerm_subnet_network_security_group_association.host,
+    azurerm_subnet_network_security_group_association.container
+  ]
+}
--- a/templates/workspace_services/databricks/terraform/network.tf
+++ b/templates/workspace_services/databricks/terraform/network.tf
@ -0,0 +1,232 @@
+resource "azurerm_network_security_group" "nsg" {
+  name                = local.network_security_group_name
+  location            = data.azurerm_resource_group.ws.location
+  resource_group_name = data.azurerm_resource_group.ws.name
+
+  tags = local.tre_workspace_service_tags
+
+  lifecycle { ignore_changes = [tags] }
+
+  security_rule {
+    name                       = "AllowInboundDatabricksWorkerNodesToCluster"
+    description                = "Required for worker nodes communication within a cluster."
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "*"
+    source_port_range          = "*"
+    destination_port_range     = "*"
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+
+  security_rule {
+    name                       = "AllowOutboundDatabricksWorkerNodesToControlPlain"
+    description                = "Required for workers communication with Databricks Webapp."
+    priority                   = 100
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "AzureDatabricks"
+  }
+
+  security_rule {
+    name                       = "AllowOutboundDatabricksWorkerNodesToAzureSQLServices"
+    description                = "Required for workers communication with Azure SQL services."
+    priority                   = 101
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "3306"
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "Sql"
+  }
+
+  security_rule {
+    name                       = "AllowOutboundDatabricksWorkerNodesToAzureStorage"
+    description                = "Required for workers communication with Azure Storage services."
+    priority                   = 102
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "Storage"
+  }
+
+  security_rule {
+    name                       = "AllowOutboundDatabricksWorkerNodesWithinACluster"
+    description                = "Required for worker nodes communication within a cluster."
+    priority                   = 103
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "*"
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+
+  security_rule {
+    name                       = "AllowOutboundWorkerNodesToAzureEventhub"
+    description                = "Required for worker communication with Azure Eventhub services."
+    priority                   = 104
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "9093"
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "EventHub"
+  }
+
+}
+
+resource "azurerm_subnet" "host" {
+  name                 = local.host_subnet_name
+  resource_group_name  = data.azurerm_resource_group.ws.name
+  virtual_network_name = data.azurerm_virtual_network.ws.name
+  address_prefixes     = [local.host_subnet_address_space]
+
+  delegation {
+    name = "db-host-vnet-integration"
+
+    service_delegation {
+      actions = [
+        "Microsoft.Network/virtualNetworks/subnets/join/action",
+        "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
+        "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action",
+      ]
+      name = "Microsoft.Databricks/workspaces"
+    }
+  }
+}
+
+resource "azurerm_subnet" "container" {
+  name                 = local.container_subnet_name
+  resource_group_name  = data.azurerm_resource_group.ws.name
+  virtual_network_name = data.azurerm_virtual_network.ws.name
+  address_prefixes     = [local.container_subnet_address_space]
+
+  delegation {
+    name = "db-container-vnet-integration"
+
+    service_delegation {
+      actions = [
+        "Microsoft.Network/virtualNetworks/subnets/join/action",
+        "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
+        "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action",
+      ]
+      name = "Microsoft.Databricks/workspaces"
+    }
+  }
+}
+
+resource "azurerm_route_table" "rt" {
+  name                          = local.route_table_name
+  location                      = data.azurerm_resource_group.ws.location
+  resource_group_name           = data.azurerm_resource_group.ws.name
+  disable_bgp_route_propagation = false
+
+  tags = local.tre_workspace_service_tags
+  lifecycle { ignore_changes = [tags] }
+
+  route {
+    name                   = "to-firewall"
+    address_prefix         = "0.0.0.0/0"
+    next_hop_type          = "VirtualAppliance"
+    next_hop_in_ip_address = data.azurerm_firewall.firewall.ip_configuration[0].private_ip_address
+  }
+}
+
+resource "azurerm_subnet_network_security_group_association" "container" {
+  subnet_id                 = azurerm_subnet.container.id
+  network_security_group_id = azurerm_network_security_group.nsg.id
+}
+
+resource "azurerm_subnet_network_security_group_association" "host" {
+  subnet_id                 = azurerm_subnet.host.id
+  network_security_group_id = azurerm_network_security_group.nsg.id
+}
+
+resource "azurerm_subnet_route_table_association" "rt_container" {
+  subnet_id      = azurerm_subnet.container.id
+  route_table_id = azurerm_route_table.rt.id
+}
+
+resource "azurerm_subnet_route_table_association" "rt_host" {
+  subnet_id      = azurerm_subnet.host.id
+  route_table_id = azurerm_route_table.rt.id
+}
+
+
+resource "azurerm_private_endpoint" "databricks_control_plane_private_endpoint" {
+  name                = "pe-adb-cp-${local.service_resource_name_suffix}"
+  location            = data.azurerm_resource_group.ws.location
+  resource_group_name = data.azurerm_resource_group.ws.name
+  subnet_id           = data.azurerm_subnet.services.id
+  tags                = local.tre_workspace_service_tags
+
+  lifecycle { ignore_changes = [tags] }
+
+  private_service_connection {
+    name                           = "private-service-connection-databricks-control-plane-${local.service_resource_name_suffix}"
+    private_connection_resource_id = azurerm_databricks_workspace.databricks.id
+    is_manual_connection           = false
+    subresource_names              = ["databricks_ui_api"]
+  }
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group-databricks-control-plane-${local.service_resource_name_suffix}"
+    private_dns_zone_ids = [data.azurerm_private_dns_zone.databricks.id]
+  }
+}
+
+resource "azurerm_private_endpoint" "databricks_auth_private_endpoint" {
+  name                = "pe-adb-auth-${local.service_resource_name_suffix}"
+  location            = data.azurerm_resource_group.ws.location
+  resource_group_name = data.azurerm_resource_group.ws.name
+  subnet_id           = data.azurerm_subnet.services.id
+  tags                = local.tre_workspace_service_tags
+
+  lifecycle { ignore_changes = [tags] }
+
+  private_service_connection {
+    name                           = "private-service-connection-databricks-auth-${local.service_resource_name_suffix}"
+    private_connection_resource_id = azurerm_databricks_workspace.databricks.id
+    is_manual_connection           = false
+    subresource_names              = ["browser_authentication"]
+  }
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group-databricks-auth-${local.service_resource_name_suffix}"
+    private_dns_zone_ids = [data.azurerm_private_dns_zone.databricks.id]
+  }
+}
+
+resource "azurerm_private_endpoint" "databricks_filesystem_private_endpoint" {
+  name                = "pe-adb-fs-${local.service_resource_name_suffix}"
+  location            = data.azurerm_resource_group.ws.location
+  resource_group_name = data.azurerm_resource_group.ws.name
+  subnet_id           = data.azurerm_subnet.services.id
+  tags                = local.tre_workspace_service_tags
+
+  lifecycle { ignore_changes = [tags] }
+
+  private_service_connection {
+    name                           = "private-service-connection-databricks-filesystem-${local.service_resource_name_suffix}"
+    private_connection_resource_id = join("", [azurerm_databricks_workspace.databricks.managed_resource_group_id, "/providers/Microsoft.Storage/storageAccounts/${local.storage_name}"])
+    is_manual_connection           = false
+    subresource_names              = ["blob"]
+  }
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group-databricks-filesystem-${local.service_resource_name_suffix}"
+    private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id]
+  }
+}
--- a/templates/workspace_services/databricks/terraform/outputs.tf
+++ b/templates/workspace_services/databricks/terraform/outputs.tf
@ -0,0 +1,55 @@
+output "databricks_workspace_name" {
+  value = azurerm_databricks_workspace.databricks.name
+}
+
+output "connection_uri" {
+  value = var.is_exposed_externally ? "https://${azurerm_databricks_workspace.databricks.workspace_url}/aad/auth?has=&Workspace=${data.azurerm_subscription.current.id}/resourceGroups/${local.resource_group_name}/providers/Microsoft.Databricks/workspaces/${local.databricks_workspace_name}&WorkspaceResourceGroupUri=${data.azurerm_subscription.current.id}/resourceGroups/${local.managed_resource_group_name}&l=en-us" : ""
+}
+
+output "internal_connection_uri" {
+  value = var.is_exposed_externally ? "" : "https://${azurerm_databricks_workspace.databricks.workspace_url}/aad/auth?has=&Workspace=${data.azurerm_subscription.current.id}/resourceGroups/${local.resource_group_name}/providers/Microsoft.Databricks/workspaces/${local.databricks_workspace_name}&WorkspaceResourceGroupUri=${data.azurerm_subscription.current.id}/resourceGroups/${local.managed_resource_group_name}&l=en-us"
+}
+
+output "databricks_storage_account_name" {
+  value = azurerm_databricks_workspace.databricks.custom_parameters[0].storage_account_name
+}
+
+output "dbfs_blob_storage_domain" {
+  value = replace("<stgacc>.blob.core.windows.net", "<stgacc>", azurerm_databricks_workspace.databricks.custom_parameters[0].storage_account_name)
+}
+
+output "log_blob_storage_domains" {
+  value = local.map_location_url_config[module.azure_region.location_cli].logBlobStorageDomains
+}
+
+output "artifact_blob_storage_domains" {
+  value = setunion(local.map_location_url_config[module.azure_region.location_cli].artifactBlobStoragePrimaryDomains, local.map_location_url_config[module.azure_region.location_cli].artifactBlobStorageSecondaryDomains)
+}
+
+output "workspace_address_spaces" {
+  value = data.azurerm_virtual_network.ws.address_space
+}
+
+output "databricks_address_prefixes" {
+  value = setunion(azurerm_subnet.container.address_prefixes, azurerm_subnet.host.address_prefixes)
+}
+
+# convert list of metastore domains to ip addresses
+data "dns_a_record_set" "metastore_addresses" {
+  for_each = toset(local.map_location_url_config[module.azure_region.location_cli].metastoreDomains)
+  host     = each.key
+}
+
+output "metastore_addresses" {
+  value = setunion(flatten([for addr in data.dns_a_record_set.metastore_addresses : addr.addrs]))
+}
+
+# convert list of event hub endpoint domains to ip addresses
+data "dns_a_record_set" "event_hub_endpoint_addresses" {
+  for_each = toset(local.map_location_url_config[module.azure_region.location_cli].eventHubEndpointDomains)
+  host     = each.key
+}
+
+output "event_hub_endpoint_addresses" {
+  value = setunion(flatten([for addr in data.dns_a_record_set.event_hub_endpoint_addresses : addr.addrs]))
+}
--- a/templates/workspace_services/databricks/terraform/providers.tf
+++ b/templates/workspace_services/databricks/terraform/providers.tf
@ -0,0 +1,59 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "=3.40.0"
+    }
+    azapi = {
+      source  = "Azure/azapi"
+      version = "=1.1.0"
+    }
+    databricks = {
+      source  = "databricks/databricks"
+      version = "=1.5.0"
+    }
+    dns = {
+      source  = "hashicorp/dns"
+      version = "=3.2.3"
+    }
+  }
+
+  backend "azurerm" {}
+}
+
+provider "azurerm" {
+  features {
+    key_vault {
+      # Don't purge on destroy (this would fail due to purge protection being enabled on keyvault)
+      purge_soft_delete_on_destroy               = false
+      purge_soft_deleted_secrets_on_destroy      = false
+      purge_soft_deleted_certificates_on_destroy = false
+      purge_soft_deleted_keys_on_destroy         = false
+      # When recreating an environment, recover any previously soft deleted secrets - set to true by default
+      recover_soft_deleted_key_vaults   = true
+      recover_soft_deleted_secrets      = true
+      recover_soft_deleted_certificates = true
+      recover_soft_deleted_keys         = true
+    }
+  }
+}
+
+provider "azapi" {
+}
+
+provider "databricks" {
+  host                        = azurerm_databricks_workspace.databricks.workspace_url
+  azure_workspace_resource_id = azurerm_databricks_workspace.databricks.id
+
+  azure_use_msi = true
+}
+
+module "azure_region" {
+  source  = "claranet/regions/azurerm"
+  version = "=6.1.0"
+
+  azure_region = data.azurerm_resource_group.ws.location
+}
+
+provider "dns" {
+}
--- a/templates/workspace_services/databricks/terraform/variables.tf
+++ b/templates/workspace_services/databricks/terraform/variables.tf
@ -0,0 +1,24 @@
+variable "tre_id" {
+  type        = string
+  description = "Unique TRE ID"
+}
+
+variable "tre_resource_id" {
+  type        = string
+  description = "Unique TRE Resource ID"
+}
+
+variable "workspace_id" {
+  type        = string
+  description = "Unique TRE WORKSPACE ID"
+}
+
+variable "address_space" {
+  type        = string
+  description = "The address space that is used by the databricks subnets."
+}
+
+variable "is_exposed_externally" {
+  type        = bool
+  description = "If the databricks workspace is exposed externally or not."
+}
--- a/templates/workspaces/base/porter.yaml
+++ b/templates/workspaces/base/porter.yaml
@ -1,7 +1,7 @@
 ---
 schemaVersion: 1.0.0
 name: tre-workspace-base
-version: 0.8.1
+version: 1.0.0
 description: "A base Azure TRE workspace"
 dockerfile: Dockerfile.tmpl
 registry: azuretre
--- a/templates/workspaces/base/terraform/network/data.tf
+++ b/templates/workspaces/base/terraform/network/data.tf
@ -53,6 +53,11 @@ data "azurerm_private_dns_zone" "blobcore" {
  resource_group_name = local.core_resource_group_name
 }

+data "azurerm_private_dns_zone" "dfscore" {
+  name                = "privatelink.dfs.core.windows.net"
+  resource_group_name = local.core_resource_group_name
+}
+
 data "azurerm_private_dns_zone" "vaultcore" {
  name                = "privatelink.vaultcore.azure.net"
  resource_group_name = local.core_resource_group_name
@ -102,3 +107,8 @@ data "azurerm_private_dns_zone" "dicom" {
  name                = "privatelink.dicom.azurehealthcareapis.com"
  resource_group_name = local.core_resource_group_name
 }
+
+data "azurerm_private_dns_zone" "databricks" {
+  name                = "privatelink.azuredatabricks.net"
+  resource_group_name = local.core_resource_group_name
+}
--- a/templates/workspaces/base/terraform/network/outputs.tf
+++ b/templates/workspaces/base/terraform/network/outputs.tf
@ -14,6 +14,10 @@ output "blobcore_zone_id" {
  value = data.azurerm_private_dns_zone.blobcore.id
 }

+output "dfscore_zone_id" {
+  value = data.azurerm_private_dns_zone.dfscore.id
+}
+
 output "airlock_processor_subnet_id" {
  value = data.azurerm_subnet.airlockprocessor.id
 }
--- a/templates/workspaces/base/terraform/network/zone_links.tf
+++ b/templates/workspaces/base/terraform/network/zone_links.tf
@ -29,6 +29,16 @@ resource "azurerm_private_dns_zone_virtual_network_link" "blobcorelink" {
  lifecycle { ignore_changes = [tags] }
 }

+resource "azurerm_private_dns_zone_virtual_network_link" "dfscorelink" {
+  name                  = "dfscorelink-${local.workspace_resource_name_suffix}"
+  resource_group_name   = local.core_resource_group_name
+  private_dns_zone_name = data.azurerm_private_dns_zone.dfscore.name
+  virtual_network_id    = azurerm_virtual_network.ws.id
+  tags                  = var.tre_workspace_tags
+
+  lifecycle { ignore_changes = [tags] }
+}
+
 resource "azurerm_private_dns_zone_virtual_network_link" "vaultcorelink" {
  name                  = "vaultcorelink-${local.workspace_resource_name_suffix}"
  resource_group_name   = local.core_resource_group_name
@ -128,3 +138,13 @@ resource "azurerm_private_dns_zone_virtual_network_link" "nexuslink" {

  lifecycle { ignore_changes = [tags] }
 }
+
+resource "azurerm_private_dns_zone_virtual_network_link" "databrickslink" {
+  name                  = "databrickslink-${local.workspace_resource_name_suffix}"
+  resource_group_name   = local.core_resource_group_name
+  private_dns_zone_name = data.azurerm_private_dns_zone.databricks.name
+  virtual_network_id    = azurerm_virtual_network.ws.id
+  tags                  = var.tre_workspace_tags
+
+  lifecycle { ignore_changes = [tags] }
+}
--- a/templates/workspaces/base/terraform/storage.tf
+++ b/templates/workspaces/base/terraform/storage.tf
@ -4,6 +4,7 @@ resource "azurerm_storage_account" "stg" {
  location                 = azurerm_resource_group.ws.location
  account_tier             = "Standard"
  account_replication_type = "GRS"
+  is_hns_enabled           = true
  tags                     = local.tre_workspace_tags

  lifecycle { ignore_changes = [tags] }
@ -20,6 +21,12 @@ resource "azurerm_storage_share" "shared_storage" {
  ]
 }

+resource "azurerm_storage_container" "stgcontainer" {
+  name                  = "datalake"
+  storage_account_name  = azurerm_storage_account.stg.name
+  container_access_type = "private"
+}
+
 resource "azurerm_storage_account_network_rules" "stgrules" {
  storage_account_id = azurerm_storage_account.stg.id

@ -79,3 +86,29 @@ resource "azurerm_private_endpoint" "stgblobpe" {
    subresource_names              = ["Blob"]
  }
 }
+
+resource "azurerm_private_endpoint" "stgdfspe" {
+  name                = "stgdfspe-${local.workspace_resource_name_suffix}"
+  location            = azurerm_resource_group.ws.location
+  resource_group_name = azurerm_resource_group.ws.name
+  subnet_id           = module.network.services_subnet_id
+  tags                = local.tre_workspace_tags
+
+  depends_on = [
+    module.network,
+  ]
+
+  lifecycle { ignore_changes = [tags] }
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [module.network.dfscore_zone_id]
+  }
+
+  private_service_connection {
+    name                           = "stgdfspesc-${local.workspace_resource_name_suffix}"
+    private_connection_resource_id = azurerm_storage_account.stg.id
+    is_manual_connection           = false
+    subresource_names              = ["dfs"]
+  }
+}