Add Databricks private authentication shared service for SSO

CHANGELOG.md

@@ -58,8 +58,11 @@ BUG FIXES:
 * Create Workspace AppInsights via AzAPI provider due to an issue with AzureRM [#3207](https://github.com/microsoft/AzureTRE/pull/3207)
 * 'Workspace Owner' is now able to access Airlock request's SAS URL even if the request is not in review [#3208](https://github.com/microsoft/AzureTRE/pull/3208)
 * Ignore changes in log_analytics_destination_type to prevent redundant updates [#3217](https://github.com/microsoft/AzureTRE/pull/3217)
+* Add Databricks private authentication shared service for SSO. [#3201](https://github.com/microsoft/AzureTRE/pull/3201)
+* Remove auth private endpoint from databricks workspace service. [3199](https://github.com/microsoft/AzureTRE/pull/3199)
 * Fix DNS conflict in airlock-review workspace that could make the entire airlock module inoperable [#3215](https://github.com/microsoft/AzureTRE/pull/3215)
 
+
 COMPONENTS:
 
 | name | version |

docs/tre-templates/workspace-services/databricks.md

@@ -2,16 +2,22 @@
 
 See: [https://azure.microsoft.com/en-us/products/databricks/](https://azure.microsoft.com/en-us/products/databricks/)
 
-This service installs the following resources into an existing virtual network within the workspace:
+This service along with Azure Databricks Private Authentication Shared Service installs the following resources into an existing virtual network within the workspace:
 
 ![Azure Databricks workspace service](../../assets/databricks_workspace_service.png)
 
 
 ## Properties
 
-- `is_exposed_externally` - If `True`, the Azure Databricks workspace is accessible from outside of the worksapce virtual network. If `False` use a Guacamole VM and the `internal_connection_uri` to access Databricks workspace.
+- `is_exposed_externally` - If `True`, the Azure Databricks workspace is accessible from outside of the workspace virtual network. If `False` use a Guacamole VM and copy the `connection_uri` to access Databricks workspace.
 
 
 ## Prerequisites
 
 - [A base workspace bundle installed](../workspaces/base.md)
+- An Azure Databricks Private Authentication Shared Service deployed - required for authenticating to an Azure Databricks workspace.
+
+
+## References
+
+- Databricks workspace service and authentication shared service deployed according to simplified deployment, for more information see: [Enable Azure Private Link as a simplified deployment](https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/private-link-simplified)

templates/shared_services/databricks-auth/.dockerignore

@@ -0,0 +1,8 @@
+# Local .terraform directories
+**/.terraform/*
+
+# TF backend files
+**/*_backend.tf
+Dockerfile.tmpl
+terraform/deploy.sh
+terraform/destroy.sh

templates/shared_services/databricks-auth/.env.sample

@@ -0,0 +1,2 @@
+ID=__CHANGE_ME__
+AZURE_LOCATION=__CHANGE_ME__

templates/shared_services/databricks-auth/Dockerfile.tmpl

@@ -0,0 +1,9 @@
+# syntax=docker/dockerfile-upstream:1.4.0
+FROM debian:bullseye-slim
+
+# PORTER_INIT
+
+# PORTER_MIXINS
+
+# Use the BUNDLE_DIR build argument to copy files into the bundle
+COPY --link . ${BUNDLE_DIR}/

templates/shared_services/databricks-auth/parameters.json

@@ -0,0 +1,38 @@
+{
+  "schemaType": "ParameterSet",
+  "schemaVersion": "1.0.1",
+  "namespace": "",
+  "name": "tre-shared-service-databricks-private-auth",
+  "parameters": [
+    {
+      "name": "id",
+      "source": {
+        "env": "ID"
+      }
+    },
+    {
+      "name": "tre_id",
+      "source": {
+        "env": "TRE_ID"
+      }
+    },
+    {
+      "name": "tfstate_container_name",
+      "source": {
+        "env": "TERRAFORM_STATE_CONTAINER_NAME"
+      }
+    },
+    {
+      "name": "tfstate_resource_group_name",
+      "source": {
+        "env": "MGMT_RESOURCE_GROUP_NAME"
+      }
+    },
+    {
+      "name": "tfstate_storage_account_name",
+      "source": {
+        "env": "MGMT_STORAGE_ACCOUNT_NAME"
+      }
+    }
+  ]
+}

templates/shared_services/databricks-auth/porter.yaml

@@ -0,0 +1,91 @@
+---
+schemaVersion: 1.0.0
+name: tre-shared-service-databricks-private-auth
+version: 0.0.6
+description: "An Azure TRE shared service for Azure Databricks autnetication."
+registry: azuretre
+dockerfile: Dockerfile.tmpl
+
+credentials:
+
+  - name: azure_tenant_id
+    env: ARM_TENANT_ID
+  - name: azure_subscription_id
+    env: ARM_SUBSCRIPTION_ID
+  - name: azure_client_id
+    env: ARM_CLIENT_ID
+  - name: azure_client_secret
+    env: ARM_CLIENT_SECRET
+
+parameters:
+  - name: tre_id
+    type: string
+  - name: id
+    type: string
+    description: "Resource ID"
+  - name: tfstate_resource_group_name
+    type: string
+    description: "Resource group containing the Terraform state storage account"
+  - name: tfstate_storage_account_name
+    type: string
+    description: "The name of the Terraform state storage account"
+  - name: tfstate_container_name
+    env: tfstate_container_name
+    type: string
+    default: "tfstate"
+    description: "The name of the Terraform state storage container"
+  - name: arm_use_msi
+    env: ARM_USE_MSI
+    type: boolean
+    default: false
+
+outputs:
+  - name: databricks_workspace_name
+    type: string
+    applyTo:
+      - install
+      - upgrade
+
+mixins:
+  - terraform:
+      clientVersion: 1.3.6
+
+install:
+  - terraform:
+      description: "Deploy Databricks Private Authentication Shared Service"
+      vars:
+        tre_resource_id: ${ bundle.parameters.id }
+        tre_id: ${ bundle.parameters.tre_id }
+      backendConfig:
+        resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
+        storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
+        container_name: ${ bundle.parameters.tfstate_container_name }
+        key: ${ bundle.name }-${ bundle.parameters.id }
+      outputs:
+        - name: databricks_workspace_name
+
+upgrade:
+  - terraform:
+      description: "Upgrade Databricks Private Authentication Shared Service"
+      vars:
+        tre_resource_id: ${ bundle.parameters.id }
+        tre_id: ${ bundle.parameters.tre_id }
+      backendConfig:
+        resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
+        storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
+        container_name: ${ bundle.parameters.tfstate_container_name }
+        key: ${ bundle.name }-${ bundle.parameters.id }
+      outputs:
+        - name: databricks_workspace_name
+
+uninstall:
+  - terraform:
+      description: "Uninstall Azure Databricks Private Authentication Shared Service"
+      vars:
+        tre_resource_id: ${ bundle.parameters.id }
+        tre_id: ${ bundle.parameters.tre_id }
+      backendConfig:
+        resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
+        storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
+        container_name: ${ bundle.parameters.tfstate_container_name }
+        key: ${ bundle.name }-${ bundle.parameters.id }

templates/shared_services/databricks-auth/template_schema.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/databricks-auth/template_schema.json",
+  "type": "object",
+  "title": "Azure Databricks Private Authentication Shared Service",
+  "description": "Azure Databricks Private Authentication Shared Service",
+  "required": [],
+  "properties": {
+    "display_name": {
+      "type": "string",
+      "title": "Name for the shared service",
+      "description": "The name of the shared service to be displayed to users",
+      "default": "Azure Databricks Private Authentication Shared Service",
+      "updateable": true
+    },
+    "description": {
+      "type": "string",
+      "title": "Description of the shared service",
+      "description": "The description of the shared service to be displayed to users",
+      "default": "Azure Databricks Private Authentication Shared Service is a Databricks workspace that you create in the same region as your Azure Databricks workspaces, and its only purpose is hosting the browser authentication private endpoint connection for your actual production Azure Databricks workspaces in that region. The private web auth workspace is not used for anything else but authentication.",
+      "updateable": true
+    },
+    "overview": {
+      "type": "string",
+      "title": "Overview for the shared service",
+      "description": "Long form description of the shared service, in markdown syntax",
+      "default": "Azure Databricks Private Authentication Shared Service is a Databricks workspace that you create in the same region as your Azure Databricks workspaces, and its only purpose is hosting the browser authentication private endpoint connection for your actual production Azure Databricks workspaces in that region. The private web auth workspace is not used for anything else but authentication.\n[Enable Azure Private Link as a simplified deployment](https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/private-link-simplified)",
+      "updateable": true
+    }
+  }
+}

templates/shared_services/databricks-auth/terraform/data.tf

@@ -0,0 +1,19 @@
+data "azurerm_resource_group" "rg" {
+  name = local.core_resource_group_name
+}
+
+data "azurerm_virtual_network" "core" {
+  name                = local.core_virtual_network_name
+  resource_group_name = data.azurerm_resource_group.rg.name
+}
+
+data "azurerm_subnet" "services" {
+  name                 = "SharedSubnet"
+  virtual_network_name = data.azurerm_virtual_network.core.name
+  resource_group_name  = data.azurerm_virtual_network.core.resource_group_name
+}
+
+data "azurerm_private_dns_zone" "databricks" {
+  name                = "privatelink.azuredatabricks.net"
+  resource_group_name = local.core_resource_group_name
+}

templates/shared_services/databricks-auth/terraform/deploy.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Uncomment this line to see each command for debugging (careful: this will show secrets!)
+#set -o xtrace
+
+export TF_LOG="TRACE"
+export TF_LOG_PATH="/home/adminuser/tf.log"
+
+# shellcheck disable=SC2154
+terraform init -input=false -backend=true -reconfigure \
+    -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \
+    -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \
+    -backend-config="container_name=$TF_VAR_terraform_state_container_name" \
+    -backend-config="key=tre-workspace-service-gitea-${TF_VAR_id}"
+
+terraform plan
+
+terraform apply -auto-approve

templates/shared_services/databricks-auth/terraform/locals.tf

@@ -0,0 +1,21 @@
+locals {
+  address_space                  = cidrsubnets("10.28.0.0/23", 1, 1)
+  container_subnet_address_space = local.address_space[0] # private
+  host_subnet_address_space      = local.address_space[1] # public
+  short_service_id               = substr(var.tre_resource_id, -4, -1)
+  service_resource_name_suffix   = "${var.tre_id}-svc-${local.short_service_id}"
+  resource_group_name            = "rg-${var.tre_id}-svc-${local.short_service_id}"
+  virtual_network_name           = "vnet-${local.service_resource_name_suffix}"
+  core_virtual_network_name      = "vnet-${var.tre_id}"
+  core_resource_group_name       = "rg-${var.tre_id}"
+  databricks_workspace_name      = "adb-${local.service_resource_name_suffix}"
+  managed_resource_group_name    = "rg-adb-${local.service_resource_name_suffix}"
+  host_subnet_name               = "adb-host-subnet-${local.service_resource_name_suffix}"
+  container_subnet_name          = "adb-container-subnet-${local.service_resource_name_suffix}"
+  network_security_group_name    = "nsg-${local.service_resource_name_suffix}"
+
+  tre_shared_service_tags = {
+    tre_id                = var.tre_id
+    tre_shared_service_id = var.tre_resource_id
+  }
+}

templates/shared_services/databricks-auth/terraform/main.tf

@@ -0,0 +1,41 @@
+resource "azurerm_resource_group" "rg" {
+  location = data.azurerm_resource_group.rg.location
+  name     = local.resource_group_name
+  tags = merge(
+    local.tre_shared_service_tags,
+    {
+      project = "Azure Trusted Research Environment",
+      source  = "https://github.com/microsoft/AzureTRE/"
+    },
+  )
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_databricks_workspace" "databricks" {
+  name                                  = local.databricks_workspace_name
+  resource_group_name                   = local.resource_group_name
+  location                              = azurerm_resource_group.rg.location
+  sku                                   = "premium"
+  managed_resource_group_name           = local.managed_resource_group_name
+  infrastructure_encryption_enabled     = true
+  public_network_access_enabled         = false
+  network_security_group_rules_required = "NoAzureDatabricksRules"
+  tags                                  = local.tre_shared_service_tags
+
+  lifecycle { ignore_changes = [tags] }
+
+  custom_parameters {
+    no_public_ip                                         = true
+    public_subnet_name                                   = azurerm_subnet.host.name
+    private_subnet_name                                  = azurerm_subnet.container.name
+    virtual_network_id                                   = azurerm_virtual_network.ws.id
+    public_subnet_network_security_group_association_id  = azurerm_subnet_network_security_group_association.host.id
+    private_subnet_network_security_group_association_id = azurerm_subnet_network_security_group_association.container.id
+  }
+
+  depends_on = [
+    azurerm_subnet_network_security_group_association.host,
+    azurerm_subnet_network_security_group_association.container
+  ]
+}