Merge pull request #174 from PerfectThymeTech/marvinbuss/fix_cost_allocation

marvinbuss · web-flow · commit 6b4172c2d6ca · 2025-03-21T10:23:27.000+01:00
Update cost allocation tags and budgets
diff --git a/databricks.tf b/databricks.tf
@@ -46,6 +46,7 @@ module "databricks_data_application" {
 
   # Service variables
   app_name                                  = each.key
+  databricks_account_id                     = var.databricks_account_id
   databricks_workspace_workspace_id         = module.core.databricks_workspace_details.engineering.workspace_id
   databricks_access_connector_id            = module.data_application[each.key].databricks_access_connector_id
   databricks_cluster_policy_library_path    = var.databricks_cluster_policy_library_path
diff --git a/modules/dataapplication/actiongroup.tf b/modules/dataapplication/actiongroup.tf
@@ -1,7 +1,7 @@
 resource "azurerm_monitor_action_group" "monitor_action_group" {
   name                = "${local.prefix}-ag001"
   resource_group_name = azurerm_resource_group.resource_group_app_monitoring.name
-  tags                = var.tags
+  tags                = local.tags
 
   enabled    = true
   short_name = substr("${local.prefix}", 0, 11)
diff --git a/modules/dataapplication/aisearch.tf b/modules/dataapplication/aisearch.tf
@@ -9,7 +9,7 @@ module "ai_search" {
 
   location                                    = var.location
   resource_group_name                         = azurerm_resource_group.resource_group_app.name
-  tags                                        = var.tags
+  tags                                        = local.tags
   search_service_name                         = "${local.prefix}-srch001"
   search_service_sku                          = var.search_service_details.sku
   search_service_semantic_search_sku          = var.search_service_details.semantic_search_sku
diff --git a/modules/dataapplication/aiservice.tf b/modules/dataapplication/aiservice.tf
@@ -10,7 +10,7 @@ module "ai_service" {
 
   location                                                = var.location
   resource_group_name                                     = azurerm_resource_group.resource_group_app.name
-  tags                                                    = var.tags
+  tags                                                    = local.tags
   cognitive_account_name                                  = "${local.prefix}-${each.key}-kv001"
   cognitive_account_kind                                  = each.value.kind
   cognitive_account_sku                                   = each.value.sku
diff --git a/modules/dataapplication/alerts.tf b/modules/dataapplication/alerts.tf
@@ -2,7 +2,7 @@ resource "azurerm_monitor_activity_log_alert" "monitor_activity_log_alert_servic
   name                = "${local.prefix}-alert-servicehealth"
   location            = "global"
   resource_group_name = azurerm_resource_group.resource_group_app_monitoring.name
-  tags                = var.tags
+  tags                = local.tags
 
   enabled     = true
   description = "Alerts for service health and maintenance events."
diff --git a/modules/dataapplication/budget.tf b/modules/dataapplication/budget.tf
@@ -13,7 +13,7 @@ resource "azurerm_consumption_budget_subscription" "consumption_budget_subscript
   }
   filter {
     dynamic "tag" {
-      for_each = var.tags
+      for_each = local.tags_cost
       iterator = item
       content {
         name     = item.key
diff --git a/modules/dataapplication/databricksaccessconnector.tf b/modules/dataapplication/databricksaccessconnector.tf
@@ -6,6 +6,6 @@ module "databricks_access_connector" {
 
   location                         = var.location
   resource_group_name              = azurerm_resource_group.resource_group_app.name
-  tags                             = var.tags
+  tags                             = local.tags
   databricks_access_connector_name = "${local.prefix}-dbac001"
 }
diff --git a/modules/dataapplication/datafactory.tf b/modules/dataapplication/datafactory.tf
@@ -10,7 +10,7 @@ module "data_factory" {
 
   location                                          = var.location
   resource_group_name                               = azurerm_resource_group.resource_group_app.name
-  tags                                              = var.tags
+  tags                                              = local.tags
   data_factory_name                                 = "${local.prefix}-adf001"
   data_factory_purview_id                           = null
   data_factory_azure_devops_repo                    = {}
diff --git a/modules/dataapplication/keyvault.tf b/modules/dataapplication/keyvault.tf
@@ -7,7 +7,7 @@ module "key_vault" {
 
   location                             = var.location
   resource_group_name                  = azurerm_resource_group.resource_group_app.name
-  tags                                 = var.tags
+  tags                                 = local.tags
   key_vault_name                       = "${local.prefix}-kv001"
   key_vault_sku_name                   = "standard"
   key_vault_soft_delete_retention_days = 7
diff --git a/modules/dataapplication/locals.tf b/modules/dataapplication/locals.tf
@@ -2,6 +2,18 @@ locals {
   # General locals
   prefix                           = "${lower(var.prefix)}-${var.environment}-${lower(var.app_name)}"
   budget_start_date_rotation_years = 9
+  tags = merge(
+    var.tags,
+    {
+      prefix      = local.prefix
+      app_name    = var.app_name
+      environment = var.environment
+    }
+  )
+  tags_cost = {
+    app_name    = var.app_name
+    cost_center = var.tags["cost_center"]
+  }
 
   # Databricks locals
   databricks_enterprise_application_id = "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d"
diff --git a/modules/dataapplication/main.tf b/modules/dataapplication/main.tf
@@ -1,11 +1,11 @@
 resource "azurerm_resource_group" "resource_group_app" {
   name     = "${local.prefix}-rg"
   location = var.location
-  tags     = var.tags
+  tags     = local.tags
 }
 
 resource "azurerm_resource_group" "resource_group_app_monitoring" {
   name     = "${local.prefix}-mntrng-rg"
   location = var.location
-  tags     = var.tags
+  tags     = local.tags
 }
diff --git a/modules/dataapplication/privateendpoint.tf b/modules/dataapplication/privateendpoint.tf
@@ -4,6 +4,7 @@ resource "azurerm_private_endpoint" "private_endpoint" {
   name                = "${local.prefix}-${each.key}-pe"
   resource_group_name = azurerm_resource_group.resource_group_app.name
   location            = var.location
+  tags                = local.tags
 
   custom_network_interface_name = "${local.prefix}-${each.key}-nic"
   private_service_connection {
diff --git a/modules/dataapplication/storagecontainers.tf b/modules/dataapplication/storagecontainers.tf
@@ -7,10 +7,7 @@ resource "azurerm_storage_container" "storage_container_external" {
   container_access_type             = "private"
   default_encryption_scope          = null
   encryption_scope_override_enabled = null
-  metadata = {
-    app_name    = var.app_name
-    environment = var.environment
-  }
+  metadata                          = local.tags
 
   depends_on = [
     var.storage_dependencies,
@@ -24,10 +21,7 @@ resource "azurerm_storage_container" "storage_container_raw" {
   container_access_type             = "private"
   default_encryption_scope          = null
   encryption_scope_override_enabled = null
-  metadata = {
-    app_name    = var.app_name
-    environment = var.environment
-  }
+  metadata                          = local.tags
 
   depends_on = [
     var.storage_dependencies,
@@ -41,10 +35,7 @@ resource "azurerm_storage_container" "storage_container_enriched" {
   container_access_type             = "private"
   default_encryption_scope          = null
   encryption_scope_override_enabled = null
-  metadata = {
-    app_name    = var.app_name
-    environment = var.environment
-  }
+  metadata                          = local.tags
 
   depends_on = [
     var.storage_dependencies,
@@ -58,10 +49,7 @@ resource "azurerm_storage_container" "storage_container_curated" {
   container_access_type             = "private"
   default_encryption_scope          = null
   encryption_scope_override_enabled = null
-  metadata = {
-    app_name    = var.app_name
-    environment = var.environment
-  }
+  metadata                          = local.tags
 
   depends_on = [
     var.storage_dependencies,
@@ -75,10 +63,7 @@ resource "azurerm_storage_container" "storage_container_workspace" {
   container_access_type             = "private"
   default_encryption_scope          = null
   encryption_scope_override_enabled = null
-  metadata = {
-    app_name    = var.app_name
-    environment = var.environment
-  }
+  metadata                          = local.tags
 
   depends_on = [
     var.storage_dependencies,
diff --git a/modules/dataapplication/userassignedidentity.tf b/modules/dataapplication/userassignedidentity.tf
@@ -6,7 +6,7 @@ module "user_assigned_identity" {
 
   location                                              = var.location
   resource_group_name                                   = azurerm_resource_group.resource_group_app.name
-  tags                                                  = var.tags
+  tags                                                  = local.tags
   user_assigned_identity_name                           = "${local.prefix}-uai001"
   user_assigned_identity_federated_identity_credentials = local.user_assigned_identity_federated_identity_credentials
 }
diff --git a/modules/databrickscore/data.tf b/modules/databrickscore/data.tf
@@ -1,3 +1,5 @@
+data "databricks_current_user" "current" {}
+
 data "databricks_mws_network_connectivity_config" "network_connectivity_config" {
   provider = databricks.account
 
@@ -9,3 +11,7 @@ data "azuread_service_principal" "service_principal_terraform_plan" {
 
   display_name = var.service_principal_name_terraform_plan
 }
+
+data "databricks_sql_warehouse" "sql_endpoint_starter" {
+  name = "Starter Warehouse"
+}
diff --git a/modules/databrickscore/roleassignments_common.tf b/modules/databrickscore/roleassignments_common.tf
@@ -0,0 +1,9 @@
+resource "databricks_permissions" "permissions_sql_endpoint_starter" {
+  sql_endpoint_id = data.databricks_sql_warehouse.sql_endpoint_starter.id
+
+  # Admin permissions
+  access_control {
+    service_principal_name = data.databricks_current_user.current.user_name
+    permission_level       = "IS_OWNER"
+  }
+}
diff --git a/modules/databrickscore/workspaceconfig.tf b/modules/databrickscore/workspaceconfig.tf
@@ -1,5 +1,6 @@
 resource "databricks_workspace_conf" "workspace_conf" {
   custom_config = {
+    # allow_downloads                                  = false
     customerApprovedWSLoginExpirationTime            = "1998-01-01T00:00:00.000Z"
     enableIpAccessLists                              = true
     enableExportNotebook                             = false
diff --git a/modules/databricksdataapplication/budget.tf b/modules/databricksdataapplication/budget.tf
@@ -31,7 +31,7 @@ resource "databricks_budget_policy" "budget_policy" {
   policy_name = "${local.prefix}-budget"
 
   custom_tags = [
-    for key, value in merge(var.tags, { prefix = local.prefix }) :
+    for key, value in local.tags :
     {
       key   = key
       value = value
diff --git a/modules/databricksdataapplication/catalog.tf b/modules/databricksdataapplication/catalog.tf
@@ -6,11 +6,9 @@ resource "databricks_catalog" "catalog_internal" {
   force_destroy                  = true
   isolation_mode                 = "ISOLATED"
   properties = merge({
-    location    = var.location
-    environment = var.environment
-    app_name    = var.app_name
-    use         = "internal"
-  }, var.tags)
+    location = var.location
+    use      = "internal"
+  }, local.tags)
   storage_root = one(databricks_external_location.external_location_curated[*].url)
 }
 
@@ -22,11 +20,9 @@ resource "databricks_catalog" "catalog_external" {
   force_destroy                  = true
   isolation_mode                 = "OPEN"
   properties = merge({
-    location    = var.location
-    environment = var.environment
-    app_name    = var.app_name
-    use         = "external"
-  }, var.tags)
+    location = var.location
+    use      = "external"
+  }, local.tags)
   storage_root = one(databricks_external_location.external_location_curated[*].url)
 }
 
diff --git a/modules/databricksdataapplication/locals.tf b/modules/databricksdataapplication/locals.tf
@@ -1,6 +1,14 @@
 locals {
   # General locals
   prefix = "${lower(var.prefix)}-${var.environment}-${lower(var.app_name)}"
+  tags = merge(
+    var.tags,
+    {
+      prefix      = local.prefix
+      app_name    = var.app_name
+      environment = var.environment
+    }
+  )
 
   # Storage locals
   storage_container_external = {
@@ -38,10 +46,7 @@ locals {
   # Create file variables
   databricks_cluster_policy_file_variables = merge(
     var.databricks_cluster_policy_file_variables,
-    var.tags,
-    {
-      prefix = local.prefix
-    }
+    local.tags,
   )
 
   # Load file content
diff --git a/modules/databricksdataapplication/roleassignments_common.tf b/modules/databricksdataapplication/roleassignments_common.tf
@@ -154,59 +154,27 @@ resource "databricks_permissions" "permissions_sql_endpoint" {
   ]
 }
 
-# resource "databricks_permissions" "permissions_budget" { # Not supported yet by provider because of missing APIs: https://github.com/databricks/terraform-provider-databricks/issues/4180
-#   budget_id = databricks_budget.budget.id
-
-#   # Admin permissions
-#   access_control {
-#     group_name       = data.databricks_group.group_admin.display_name
-#     permission_level = "CAN_USE"
-#   }
-#   # Developer permissions
-#   dynamic "access_control" {
-#     for_each = var.developer_group_name == "" ? [] : [1]
-#     content {
-#       group_name       = one(data.databricks_group.group_developer[*].display_name)
-#       permission_level = "CAN_USE"
-#     }
-#   }
-#   # Reader permissions
-#   dynamic "access_control" {
-#     for_each = var.reader_group_name == "" ? [] : [1]
-#     content {
-#       group_name       = one(data.databricks_group.group_reader[*].display_name)
-#       permission_level = "CAN_USE"
-#     }
-#   }
-#   # Service principal permissions
-#   dynamic "access_control" {
-#     for_each = var.service_principal_name == "" ? [] : [1]
-#     content {
-#       service_principal_name = one(databricks_service_principal.service_principal[*].application_id)
-#       permission_level       = "CAN_USE"
-#     }
-#   }
-#   # Service principal data factory permissions
-#   dynamic "access_control" {
-#     for_each = var.databricks_data_factory_details.data_factory_enabled ? [1] : [0]
-#     content {
-#       service_principal_name = one(databricks_service_principal.service_principal_data_factory[*].application_id)
-#       permission_level       = "CAN_USE"
-#     }
-#   }
-#   # UAI permissions
-#   access_control {
-#     service_principal_name = databricks_service_principal.service_principal_uai.application_id
-#     permission_level       = "CAN_USE"
-#   }
+resource "databricks_access_control_rule_set" "access_control_rule_set_budget_policy" {
+  name = "accounts/${var.databricks_account_id}/budgetPolicies/${databricks_budget_policy.budget_policy.policy_id}/ruleSets/default"
 
+  grant_rules {
+    principals = compact([
+      data.databricks_group.group_admin.acl_principal_id,
+      one(data.databricks_group.group_developer[*].acl_principal_id),
+      one(data.databricks_group.group_reader[*].acl_principal_id),
+      one(databricks_service_principal.service_principal[*].acl_principal_id),
+      one(databricks_service_principal.service_principal_data_factory[*].acl_principal_id),
+      databricks_service_principal.service_principal_uai.acl_principal_id,
+    ])
+    role = "roles/budgetPolicy.user"
+  }
 
-#   depends_on = [
-#     databricks_permission_assignment.permission_assignment_admin,
-#     databricks_permission_assignment.permission_assignment_developer,
-#     databricks_permission_assignment.permission_assignment_reader,
-#     databricks_permission_assignment.permission_assignment_service_principal,
-#     databricks_permission_assignment.permission_assignment_service_principal_data_factory,
-#     databricks_permission_assignment.permission_assignment_uai,
-#   ]
-# }
+  depends_on = [
+    databricks_permission_assignment.permission_assignment_admin,
+    databricks_permission_assignment.permission_assignment_developer,
+    databricks_permission_assignment.permission_assignment_reader,
+    databricks_permission_assignment.permission_assignment_service_principal,
+    databricks_permission_assignment.permission_assignment_service_principal_data_factory,
+    databricks_permission_assignment.permission_assignment_uai,
+  ]
+}
diff --git a/modules/databricksdataapplication/sqlendpoint.tf b/modules/databricksdataapplication/sqlendpoint.tf
@@ -9,14 +9,14 @@ resource "databricks_sql_endpoint" "sql_endpoint" {
   }
   cluster_size              = each.value.cluster_size
   enable_photon             = true
-  enable_serverless_compute = false
+  enable_serverless_compute = each.value.enable_serverless_compute
   max_num_clusters          = each.value.max_num_clusters
   min_num_clusters          = each.value.min_num_clusters
   warehouse_type            = "PRO"
   spot_instance_policy      = "RELIABILITY_OPTIMIZED"
   tags {
     dynamic "custom_tags" {
-      for_each = merge(var.tags, { prefix = local.prefix })
+      for_each = local.tags
       iterator = entry
       content {
         key   = entry.key
diff --git a/modules/databricksdataapplication/variables.tf b/modules/databricksdataapplication/variables.tf
diff --git a/schemas/app.schema.json b/schemas/app.schema.json
diff --git a/tests/e2e/data-applications/app001.yml b/tests/e2e/data-applications/app001.yml
diff --git a/tests/e2e/databricks-cluster-policies/dlt-policy.yml b/tests/e2e/databricks-cluster-policies/dlt-policy.yml
diff --git a/tests/e2e/databricks-cluster-policies/job-policy.yml b/tests/e2e/databricks-cluster-policies/job-policy.yml
diff --git a/tests/e2e/databricks-cluster-policies/personal-compute-policy.yml b/tests/e2e/databricks-cluster-policies/personal-compute-policy.yml

Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@ resource "azurerm_consumption_budget_subscription" "consumption_budget_subscript`
`13`	`13`	`}`
`14`	`14`	`filter {`
`15`	`15`	`dynamic "tag" {`
`16`		`- for_each = var.tags`
	`16`	`+ for_each = local.tags_cost`
`17`	`17`	`iterator = item`
`18`	`18`	`content {`
`19`	`19`	`name = item.key`
Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,6 @@ module "databricks_access_connector" {`
`6`	`6`
`7`	`7`	`location = var.location`
`8`	`8`	`resource_group_name = azurerm_resource_group.resource_group_app.name`
`9`		`- tags = var.tags`
	`9`	`+ tags = local.tags`
`10`	`10`	`databricks_access_connector_name = "${local.prefix}-dbac001"`
`11`	`11`	`}`