From 292eac2f5abdcf87b1cd69e7069e3ae7ae10bfb5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 10:58:58 +0000 Subject: [PATCH 01/44] add cluster policy basics to repo --- bundle/config/resources.go | 3 ++ bundle/config/resources/cluster_policy.go | 58 +++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 bundle/config/resources/cluster_policy.go diff --git a/bundle/config/resources.go b/bundle/config/resources.go index ab12ec9f052..a77c4c5034c 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -45,6 +45,7 @@ type Resources struct { VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` Secrets map[string]*resources.Secret `json:"secrets,omitempty"` + ClusterPolicies map[string]*resources.ClusterPolicy `json:cluster_policies,omitempty` } type ConfigResource interface { @@ -131,6 +132,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), collectResourceMap(descriptions["secrets"], r.Secrets), + collectResourceMap(descriptions["cluster_policies"], r.ClusterPolicies), } } @@ -195,5 +197,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), "secrets": (&resources.Secret{}).ResourceDescription(), + "cluster_policies": (&resources.ClusterPolicy{}).ResourceDescription(), } } diff --git a/bundle/config/resources/cluster_policy.go b/bundle/config/resources/cluster_policy.go new file mode 100644 index 00000000000..cc636fe2f58 --- /dev/null +++ b/bundle/config/resources/cluster_policy.go @@ -0,0 +1,58 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ClusterPolicy struct { + BaseResource + compute.CreatePolicy +} + +func (s *ClusterPolicy) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s ClusterPolicy) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *ClusterPolicy) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.ClusterPolicies.GetByPolicyId(ctx, id) + if err != nil { + log.Debugf(ctx, "cluster policy %s does not exist", id) + return false, err + } + return true, nil +} + +func (*ClusterPolicy) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "cluster_policy", + PluralName: "cluster_policies", + SingularTitle: "Cluster Policy", + PluralTitle: "Cluster Policies", + } +} + +func (s *ClusterPolicy) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "cluster_policies", s.ID) +} + +func (s *ClusterPolicy) GetName() string { + return s.Name +} + +func (s *ClusterPolicy) GetURL() string { + return s.URL +} From 577efbf35a417deddffacc4d63bc57894091f50e Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 11:30:10 +0000 Subject: [PATCH 02/44] fix double quotes --- bundle/config/resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/resources.go b/bundle/config/resources.go index a77c4c5034c..633332a78c0 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -45,7 +45,7 @@ type Resources struct { VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` Secrets map[string]*resources.Secret `json:"secrets,omitempty"` - ClusterPolicies map[string]*resources.ClusterPolicy `json:cluster_policies,omitempty` + ClusterPolicies map[string]*resources.ClusterPolicy `json:"cluster_policies,omitempty"` } type ConfigResource interface { From 44b2393c8a8f14dd709aa8f3bb7953ad95368dfa Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 12:15:01 +0000 Subject: [PATCH 03/44] add CRUD policies --- bundle/direct/dresources/cluster_policy.go | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 bundle/direct/dresources/cluster_policy.go diff --git a/bundle/direct/dresources/cluster_policy.go b/bundle/direct/dresources/cluster_policy.go new file mode 100644 index 00000000000..f6c0db22569 --- /dev/null +++ b/bundle/direct/dresources/cluster_policy.go @@ -0,0 +1,68 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ResourceClusterPolicy struct { + client *databricks.WorkspaceClient +} + +func (*ResourceClusterPolicy) New(client *databricks.WorkspaceClient) *ResourceClusterPolicy { + return &ResourceClusterPolicy{client: client} +} + +func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *compute.CreatePolicy { + return &input.CreatePolicy +} + +// RemapState copies the config fields shared by Policy and CreatePolicy; +// output-only fields (policy_id, created_at_timestamp, creator_user_name, is_default) are not in the state. +func (*ResourceClusterPolicy) RemapState(remote *compute.Policy) *compute.CreatePolicy { + return &compute.CreatePolicy{ + Definition: remote.Definition, + Description: remote.Description, + Libraries: remote.Libraries, + MaxClustersPerUser: remote.MaxClustersPerUser, + Name: remote.Name, + PolicyFamilyDefinitionOverrides: remote.PolicyFamilyDefinitionOverrides, + PolicyFamilyId: remote.PolicyFamilyId, + ForceSendFields: utils.FilterFields[compute.CreatePolicy](remote.ForceSendFields), + } +} + +func (r *ResourceClusterPolicy) DoRead(ctx context.Context, id string) (*compute.Policy, error) { + return r.client.ClusterPolicies.GetByPolicyId(ctx, id) +} + +func (r *ResourceClusterPolicy) DoCreate(ctx context.Context, config *compute.CreatePolicy) (string, *compute.Policy, error) { + resp, err := r.client.ClusterPolicies.Create(ctx, *config) + if err != nil { + return "", nil, err + } + + return resp.PolicyId, nil, nil +} + +func (r *ResourceClusterPolicy) DoUpdate(ctx context.Context, id string, config *compute.CreatePolicy, _ *PlanEntry) (*compute.Policy, error) { + return nil, r.client.ClusterPolicies.Edit(ctx, compute.EditPolicy{ + PolicyId: id, + Name: config.Name, + Definition: config.Definition, + Description: config.Description, + Libraries: config.Libraries, + MaxClustersPerUser: config.MaxClustersPerUser, + PolicyFamilyDefinitionOverrides: config.PolicyFamilyDefinitionOverrides, + PolicyFamilyId: config.PolicyFamilyId, + ForceSendFields: utils.FilterFields[compute.EditPolicy](config.ForceSendFields), + }) +} + +func (r *ResourceClusterPolicy) DoDelete(ctx context.Context, id string, _ *compute.CreatePolicy) error { + return r.client.ClusterPolicies.DeleteByPolicyId(ctx, id) +} From f5a668f283fe97afab3835b9c0fe7f6a5f711267 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 12:20:12 +0000 Subject: [PATCH 04/44] register cluster policies in all.go --- bundle/direct/dresources/all.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ad310468da0..2c82df2aab9 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -40,6 +40,7 @@ var SupportedResources = map[string]any{ "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), "instance_pools": (*ResourceInstancePool)(nil), "secrets": (*ResourceSecret)(nil), + "cluster_policies": (*ResourceClusterPolicy)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), From bc3de96c4eae79593732152cbdf996ae6e80b1fd Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 12:42:52 +0000 Subject: [PATCH 05/44] add cluster policies to the testserver --- libs/testserver/cluster_policies.go | 80 +++++++++++++++++++++++++++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 7 +++ 3 files changed, 89 insertions(+) create mode 100644 libs/testserver/cluster_policies.go diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go new file mode 100644 index 00000000000..61c69347baf --- /dev/null +++ b/libs/testserver/cluster_policies.go @@ -0,0 +1,80 @@ +package testserver + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) ClusterPoliciesCreate(req Request) any { + // Unmarshal into the stored (GET) type directly: CreatePolicy and Policy + // share JSON field names, so every config field is carried over. + var policy compute.Policy + if err := json.Unmarshal(req.Body, &policy); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + id := nextUUID() + policy.PolicyId = id + s.ClusterPolicies[id] = policy + + return Response{Body: compute.CreatePolicyResponse{PolicyId: id}} +} + +func (s *FakeWorkspace) ClusterPoliciesGet(req Request, policyId string) any { + defer s.LockUnlock()() + + policy, ok := s.ClusterPolicies[policyId] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: policy} +} + +func (s *FakeWorkspace) ClusterPoliciesEdit(req Request) any { + var request compute.EditPolicy + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + policy, ok := s.ClusterPolicies[request.PolicyId] + if !ok { + return Response{StatusCode: 404} + } + + // Edit is a full replace of the writable fields; server-set fields + // (policy_id, created_at_timestamp, creator_user_name, is_default) are kept as stored. + policy.Name = request.Name + policy.Definition = request.Definition + policy.Description = request.Description + policy.Libraries = request.Libraries + policy.MaxClustersPerUser = request.MaxClustersPerUser + policy.PolicyFamilyDefinitionOverrides = request.PolicyFamilyDefinitionOverrides + policy.PolicyFamilyId = request.PolicyFamilyId + s.ClusterPolicies[request.PolicyId] = policy + + return Response{} +} + +func (s *FakeWorkspace) ClusterPoliciesDelete(req Request) any { + var request compute.DeletePolicy + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.ClusterPolicies[request.PolicyId]; !ok { + return Response{StatusCode: 404} + } + + delete(s.ClusterPolicies, request.PolicyId) + + return Response{} +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 445417b3ff1..9dbca5373e5 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -196,6 +196,7 @@ type FakeWorkspace struct { ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails InstancePools map[string]compute.GetInstancePool + ClusterPolicies map[string]compute.Policy Catalogs map[string]catalog.CatalogInfo ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo @@ -429,6 +430,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { }, }, InstancePools: map[string]compute.GetInstancePool{}, + ClusterPolicies: map[string]compute.Policy{}, VectorSearchIndexesPendingDeletion: map[string]int{}, } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 082b771d1ea..9d2f765b716 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -62,6 +62,13 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.InstancePoolsGet(req, req.URL.Query().Get("instance_pool_id")) }) + server.Handle("POST", "/api/2.0/policies/clusters/create", func(req Request) any { return req.Workspace.ClusterPoliciesCreate(req) }) + server.Handle("POST", "/api/2.0/policies/clusters/edit", func(req Request) any { return req.Workspace.ClusterPoliciesEdit(}) + server.Handle("POST", "/api/2.0/policies/clusters/delete", func(req Request) any { return req.Workspace.ClusterPoliciesDelete(req) }) + server.Handle("GET", "/api/2.0/policies/clusters/get", func(req Request) any { + return req.Workspace.ClusterPoliciesGet(req, req.URL.Query().Get("policy_id")) + }) + server.Handle("GET", "/api/2.1/clusters/list", func(req Request) any { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ From 2202fd086261c32d41add203675eadf42f06dca1 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 12:48:22 +0000 Subject: [PATCH 06/44] fix syntax error --- libs/testserver/handlers.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 9d2f765b716..282dbec9338 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -63,11 +63,11 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/policies/clusters/create", func(req Request) any { return req.Workspace.ClusterPoliciesCreate(req) }) - server.Handle("POST", "/api/2.0/policies/clusters/edit", func(req Request) any { return req.Workspace.ClusterPoliciesEdit(}) - server.Handle("POST", "/api/2.0/policies/clusters/delete", func(req Request) any { return req.Workspace.ClusterPoliciesDelete(req) }) - server.Handle("GET", "/api/2.0/policies/clusters/get", func(req Request) any { - return req.Workspace.ClusterPoliciesGet(req, req.URL.Query().Get("policy_id")) - }) + server.Handle("POST", "/api/2.0/policies/clusters/edit", func(req Request) any { return req.Workspace.ClusterPoliciesEdit(req) }) + server.Handle("POST", "/api/2.0/policies/clusters/delete", func(req Request) any { return req.Workspace.ClusterPoliciesDelete(req) }) + server.Handle("GET", "/api/2.0/policies/clusters/get", func(req Request) any { + return req.Workspace.ClusterPoliciesGet(req, req.URL.Query().Get("policy_id")) + }) server.Handle("GET", "/api/2.1/clusters/list", func(req Request) any { return compute.ListClustersResponse{ From d0aeb0e762099d101011721f334ad236b52eac29 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 12:49:09 +0000 Subject: [PATCH 07/44] add generated bundle files --- bundle/internal/schema/annotations.yml | 7 +++ bundle/schema/jsonschema.json | 62 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 60bedb38d16..bd1c02857ad 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -543,6 +543,13 @@ resources: "azure_tenant_id": "description": |- PLACEHOLDER + "cluster_policies": + "description": |- + PLACEHOLDER + "$fields": + "lifecycle": + "description": |- + PLACEHOLDER "clusters": "description": |- The cluster definitions for the bundle, where each key is the name of a cluster. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index b4084979746..e136dab10eb 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -589,6 +589,51 @@ } ] }, + "resources.ClusterPolicy": { + "oneOf": [ + { + "type": "object", + "properties": { + "definition": { + "description": "Policy definition document expressed in [Databricks Cluster Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).", + "$ref": "#/$defs/string" + }, + "description": { + "description": "Additional human-readable description of the cluster policy.", + "$ref": "#/$defs/string" + }, + "libraries": { + "description": "A list of libraries to be installed on the next cluster restart that uses this policy. The maximum number of libraries is 500.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "max_clusters_per_user": { + "description": "Max number of clusters per user that can be active using this policy. If not present, there is no max limit.", + "$ref": "#/$defs/int64" + }, + "name": { + "description": "Cluster Policy name requested by the user. This has to be unique. Length must be between 1 and 100\ncharacters.", + "$ref": "#/$defs/string" + }, + "policy_family_definition_overrides": { + "description": "Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).\nThe JSON document must be passed as a string and cannot be embedded in the requests.\n\nYou can use this to customize the policy definition inherited from the policy family.\nPolicy rules specified here are merged into the inherited policy definition.", + "$ref": "#/$defs/string" + }, + "policy_family_id": { + "description": "ID of the policy family. The cluster policy's policy definition inherits the policy\nfamily's policy definition.\n\nCannot be used with `definition`. Use `policy_family_definition_overrides` instead to\ncustomize the policy definition.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Dashboard": { "oneOf": [ { @@ -3356,6 +3401,9 @@ "catalogs": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Catalog" }, + "cluster_policies": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ClusterPolicy" + }, "clusters": { "description": "The cluster definitions for the bundle, where each key is the name of a cluster.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster", @@ -14573,6 +14621,20 @@ } ] }, + "resources.ClusterPolicy": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ClusterPolicy" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Dashboard": { "oneOf": [ { From 368fbb8e5b5c40ff7c5e65ef2829eadf4de7a2c6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 12:49:44 +0000 Subject: [PATCH 08/44] add acceptance tests for cluster policies --- .../resources/cluster_policies/databricks.yml | 8 ++ .../resources/cluster_policies/out.test.toml | 2 + .../resources/cluster_policies/output.txt | 126 ++++++++++++++++++ .../bundle/resources/cluster_policies/script | 33 +++++ .../resources/cluster_policies/test.toml | 6 + 5 files changed, 175 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/script create mode 100644 acceptance/bundle/resources/cluster_policies/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/databricks.yml b/acceptance/bundle/resources/cluster_policies/databricks.yml new file mode 100644 index 00000000000..5156a2b9b0f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test_cluster_policy + +resources: + cluster_policies: + test_cluster_policy: + name: my_cluster_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/out.test.toml b/acceptance/bundle/resources/cluster_policies/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/output.txt b/acceptance/bundle/resources/cluster_policies/output.txt new file mode 100644 index 00000000000..97b27c42611 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/output.txt @@ -0,0 +1,126 @@ + +>>> [CLI] bundle validate +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Validation OK! + +>>> [CLI] bundle validate -o json +{ + "test_cluster_policy": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the create request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/create"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy + URL: (not deployed) + +=== Update the cluster policy name +>>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the update request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy_2", + "policy_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy_2 + URL: (not deployed) + +=== Destroy the cluster policy +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.test_cluster_policy + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Deleting files... +Destroy complete! + +=== Verify the destroy request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/delete"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/delete", + "body": { + "policy_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy_2 + URL: (not deployed) + +>>> [CLI] bundle destroy --auto-approve +No active deployment found to destroy! diff --git a/acceptance/bundle/resources/cluster_policies/script b/acceptance/bundle/resources/cluster_policies/script new file mode 100644 index 00000000000..412bdcf1520 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/script @@ -0,0 +1,33 @@ +trace $CLI bundle validate +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle summary + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT +trace $CLI bundle deploy + +title "Verify the create request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/create")))' out.requests.txt + +trace $CLI bundle summary + +title "Update the cluster policy name" +trace update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 +trace $CLI bundle deploy + +title "Verify the update request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt + +trace $CLI bundle summary + +title "Destroy the cluster policy" +trace $CLI bundle destroy --auto-approve + +title "Verify the destroy request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/delete")))' out.requests.txt + +trace $CLI bundle summary diff --git a/acceptance/bundle/resources/cluster_policies/test.toml b/acceptance/bundle/resources/cluster_policies/test.toml new file mode 100644 index 00000000000..3fe510e7c1b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/test.toml @@ -0,0 +1,6 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] From ddd5bdbac05df72b5c562f3f388218d31142a26c Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 12:59:23 +0000 Subject: [PATCH 09/44] add cluster_policies to mutator --- .../mutator/resourcemutator/run_as_test.go | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 87312608de8..be9abfe6836 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -31,41 +31,43 @@ func allResourceTypes(t *testing.T) []string { // Assert the total list of resource supported, as a sanity check that using // the dyn library gives us the correct list of all resources supported. Please // also update this check when adding a new resource - require.Equal(t, []string{ - "alerts", - "apps", - "catalogs", - "clusters", - "dashboards", - "database_catalogs", - "database_instances", - "experiments", - "external_locations", - "genie_spaces", - "instance_pools", - "job_runs", - "jobs", - "model_serving_endpoints", - "models", - "pipelines", - "postgres_branches", - "postgres_catalogs", - "postgres_databases", - "postgres_endpoints", - "postgres_projects", - "postgres_roles", - "postgres_synced_tables", - "quality_monitors", - "registered_models", - "schemas", - "secret_scopes", - "secrets", - "sql_warehouses", - "synced_database_tables", - "vector_search_endpoints", - "vector_search_indexes", - "volumes", - }, + require.Equal( + t, []string{ + "alerts", + "apps", + "catalogs", + "cluster_policies", + "clusters", + "dashboards", + "database_catalogs", + "database_instances", + "experiments", + "external_locations", + "genie_spaces", + "instance_pools", + "job_runs", + "jobs", + "model_serving_endpoints", + "models", + "pipelines", + "postgres_branches", + "postgres_catalogs", + "postgres_databases", + "postgres_endpoints", + "postgres_projects", + "postgres_roles", + "postgres_synced_tables", + "quality_monitors", + "registered_models", + "schemas", + "secret_scopes", + "secrets", + "sql_warehouses", + "synced_database_tables", + "vector_search_endpoints", + "vector_search_indexes", + "volumes", + }, resourceTypes, ) @@ -174,6 +176,7 @@ var allowList = []string{ "alerts", "catalogs", "clusters", + "cluster_policies", "dashboards", "database_catalogs", "database_instances", From 8bcb4624881cf5ba3ee789a6bd3b664778e66b03 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 13:30:23 +0000 Subject: [PATCH 10/44] add URL to for policy to output --- acceptance/bundle/resources/cluster_policies/output.txt | 4 ++-- libs/workspaceurls/urls.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/output.txt b/acceptance/bundle/resources/cluster_policies/output.txt index 97b27c42611..3d0358684e4 100644 --- a/acceptance/bundle/resources/cluster_policies/output.txt +++ b/acceptance/bundle/resources/cluster_policies/output.txt @@ -55,7 +55,7 @@ Resources: Cluster Policies: test_cluster_policy: Name: my_cluster_policy - URL: (not deployed) + URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] === Update the cluster policy name >>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 @@ -88,7 +88,7 @@ Resources: Cluster Policies: test_cluster_policy: Name: my_cluster_policy_2 - URL: (not deployed) + URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] === Destroy the cluster policy >>> [CLI] bundle destroy --auto-approve diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 4839c1e4273..61c3f271468 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -11,6 +11,7 @@ var resourceURLPatterns = map[string]string{ "alerts": "sql/alerts-v2/%s", "apps": "apps/%s", "catalogs": "explore/data/%s", + "cluster_policies": "compute/policies/%s", "clusters": "compute/clusters/%s", "dashboards": "dashboardsv3/%s/published", "database_catalogs": "explore/data/%s", From cd13a39a17ff6b868c5ad93bacd08e4adcb473e0 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 13:53:10 +0000 Subject: [PATCH 11/44] fix tests --- .../resourcemutator/apply_bundle_permissions_test.go | 1 + bundle/config/mutator/resourcemutator/apply_presets.go | 10 ++++++++++ .../mutator/resourcemutator/apply_target_mode_test.go | 3 +++ 3 files changed, 14 insertions(+) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 46262ba8dbf..99ac6759ee8 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -36,6 +36,7 @@ var unsupportedResources = []string{ "vector_search_indexes", "job_runs", "secrets", + "cluster_policies", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 72817d13566..f31866a6fc3 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -321,6 +321,16 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } + // Cluster Policies: Prefix. The policy name is a user-facing display name + // (unique, 1-100 chars), not the API id (policy_id), so prefixing it in dev + // mode avoids collisions between developers without changing identity. + for _, cp := range r.ClusterPolicies { + if cp == nil { + continue + } + cp.Name = prefix + cp.Name + } + // Vector Search Endpoints: no prefix. The endpoint name is the primary key // (it's what GET/UPDATE/DELETE address by), so prefixing it would change // the resource's identity rather than just its display name. diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 53fdf89f50e..35eed05cc53 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -153,6 +153,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { InstancePools: map[string]*resources.InstancePool{ "instance_pool1": {CreateInstancePool: compute.CreateInstancePool{InstancePoolName: "instance_pool1", NodeTypeId: "i3.xlarge"}}, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "cluster_policy1": {CreatePolicy: compute.CreatePolicy{Name: "cluster_policy1"}}, + }, Dashboards: map[string]*resources.Dashboard{ "dashboard1": { DashboardConfig: resources.DashboardConfig{ From 61405c8832a05a1c5118864f0665f71f78b8067f Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 13:58:22 +0000 Subject: [PATCH 12/44] make cluster policies direct deployments only --- bundle/deploy/terraform/lifecycle_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index f8a5140576f..eaf3040f1c7 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -16,6 +16,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { // Resources that are only supported in direct mode and should not be converted to Terraform ignoredResources := []string{ "catalogs", + "cluster_policies", "external_locations", "genie_spaces", "instance_pools", From 77de13bb4001c6e3a91bcdae52e95e8de85884c0 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:07:27 +0000 Subject: [PATCH 13/44] add cluster policy test everywhere --- bundle/statemgmt/state_load_test.go | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index b706b0770cf..2c2bc1bc9dd 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -59,6 +59,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.secrets.test_secret": {ID: "main.default.test_secret"}, + "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -154,6 +155,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Equal(t, "cp-1", config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret"].ModifiedStatus) @@ -402,6 +406,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "test_cluster_policy": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy", + }, + }, + }, }, } @@ -507,6 +518,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.InstancePools["test_instance_pool"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -914,6 +928,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "test_cluster_policy": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy", + }, + }, + "test_cluster_policy_new": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy_new", + }, + }, + }, }, } state := ExportedResourcesMap{ @@ -973,6 +999,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.instance_pools.test_instance_pool_old": {ID: "2"}, + "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, + "resources.cluster_policies.test_cluster_policy_old": {ID: "cp-2"}, "resources.secrets.test_secret": {ID: "main.default.test_secret"}, "resources.secrets.test_secret_old": {ID: "main.default.test_secret_old"}, } @@ -1177,6 +1205,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.InstancePools["test_instance_pool_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool_new"].ModifiedStatus) + assert.Equal(t, "cp-1", config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + assert.Equal(t, "cp-2", config.Resources.ClusterPolicies["test_cluster_policy_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ClusterPolicies["test_cluster_policy_old"].ModifiedStatus) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ClusterPolicies["test_cluster_policy_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) assert.Empty(t, config.Resources.Secrets["test_secret"].ModifiedStatus) assert.Equal(t, "main.default.test_secret_old", config.Resources.Secrets["test_secret_old"].ID) From e9c9eb46491c192853836ad1c31211681e1a8dda Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:12:30 +0000 Subject: [PATCH 14/44] Add bind tests --- bundle/config/resources_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d83f4da59b8..7e56f47a64a 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -204,6 +204,9 @@ func TestResourcesBindSupport(t *testing.T) { InstancePools: map[string]*resources.InstancePool{ "my_instance_pool": {}, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "my_cluster_policy": {}, + }, Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, @@ -366,6 +369,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockInstancePoolsAPI().EXPECT().GetByInstancePoolId(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockClusterPoliciesAPI().EXPECT().GetByPolicyId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) From 896f47ebc857196496e358074608ea1eb6cf1983 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:17:11 +0000 Subject: [PATCH 15/44] fix linting --- bundle/statemgmt/state_load_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 2c2bc1bc9dd..ab39a4917b0 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -999,8 +999,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.instance_pools.test_instance_pool_old": {ID: "2"}, - "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, - "resources.cluster_policies.test_cluster_policy_old": {ID: "cp-2"}, + "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, + "resources.cluster_policies.test_cluster_policy_old": {ID: "cp-2"}, "resources.secrets.test_secret": {ID: "main.default.test_secret"}, "resources.secrets.test_secret_old": {ID: "main.default.test_secret_old"}, } From 1c6ae4db19a838d868e2c794f303fd596e4a4b12 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:17:19 +0000 Subject: [PATCH 16/44] add changelog --- .nextchanges/bundles/cluster-policies.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/cluster-policies.md diff --git a/.nextchanges/bundles/cluster-policies.md b/.nextchanges/bundles/cluster-policies.md new file mode 100644 index 00000000000..ace7f4b426b --- /dev/null +++ b/.nextchanges/bundles/cluster-policies.md @@ -0,0 +1 @@ +Add support for the `cluster_policies` resource type in Declarative Automation Bundles. Cluster policies are only supported in direct deployment mode. From 01928b4697e5dc41b2039af751bfdd34be90ff91 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:25:19 +0000 Subject: [PATCH 17/44] regenerate schema files for cluster_policies The cluster_policies resource was added without regenerating derived files, failing validate-generated and the refschema acceptance test. Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 32 +++++++++++++++++++ .../direct/dresources/apitypes.generated.yml | 2 ++ .../direct/dresources/resources.generated.yml | 2 ++ .../validation/generated/required_fields.go | 4 +++ 4 files changed, 40 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 6c2b033fa97..a807286ffe0 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -279,6 +279,38 @@ resources.catalogs.*.grants[*] catalog.PrivilegeAssignment ALL resources.catalogs.*.grants[*].principal string ALL resources.catalogs.*.grants[*].privileges []catalog.Privilege ALL resources.catalogs.*.grants[*].privileges[*] catalog.Privilege ALL +resources.cluster_policies.*.created_at_timestamp int64 REMOTE +resources.cluster_policies.*.creator_user_name string REMOTE +resources.cluster_policies.*.definition string ALL +resources.cluster_policies.*.description string ALL +resources.cluster_policies.*.id string INPUT +resources.cluster_policies.*.is_default bool REMOTE +resources.cluster_policies.*.libraries []compute.Library ALL +resources.cluster_policies.*.libraries[*] compute.Library ALL +resources.cluster_policies.*.libraries[*].cran *compute.RCranLibrary ALL +resources.cluster_policies.*.libraries[*].cran.package string ALL +resources.cluster_policies.*.libraries[*].cran.repo string ALL +resources.cluster_policies.*.libraries[*].egg string ALL +resources.cluster_policies.*.libraries[*].jar string ALL +resources.cluster_policies.*.libraries[*].maven *compute.MavenLibrary ALL +resources.cluster_policies.*.libraries[*].maven.coordinates string ALL +resources.cluster_policies.*.libraries[*].maven.exclusions []string ALL +resources.cluster_policies.*.libraries[*].maven.exclusions[*] string ALL +resources.cluster_policies.*.libraries[*].maven.repo string ALL +resources.cluster_policies.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL +resources.cluster_policies.*.libraries[*].pypi.package string ALL +resources.cluster_policies.*.libraries[*].pypi.repo string ALL +resources.cluster_policies.*.libraries[*].requirements string ALL +resources.cluster_policies.*.libraries[*].whl string ALL +resources.cluster_policies.*.lifecycle resources.Lifecycle INPUT +resources.cluster_policies.*.lifecycle.prevent_destroy bool INPUT +resources.cluster_policies.*.max_clusters_per_user int64 ALL +resources.cluster_policies.*.modified_status string INPUT +resources.cluster_policies.*.name string ALL +resources.cluster_policies.*.policy_family_definition_overrides string ALL +resources.cluster_policies.*.policy_family_id string ALL +resources.cluster_policies.*.policy_id string REMOTE +resources.cluster_policies.*.url string INPUT resources.clusters.*.apply_policy_default_values bool ALL resources.clusters.*.autoscale *compute.AutoScale ALL resources.clusters.*.autoscale.max_workers int ALL diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 5e61d803183..ec2e3c2519c 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -6,6 +6,8 @@ apps: apps.App catalogs: catalog.CreateCatalog +cluster_policies: compute.CreatePolicy + clusters: compute.ClusterSpec dashboards: dashboards.Dashboard diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index d5fb0d6c98d..bf63d30dc95 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -70,6 +70,8 @@ resources: # catalogs: no api field behaviors + # cluster_policies: no api field behaviors + # clusters: no api field behaviors dashboards: diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 9268be0985f..97fa418097c 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -40,6 +40,10 @@ var RequiredFields = map[string][]string{ "resources.catalogs.*": {"name"}, "resources.catalogs.*.managed_encryption_settings.azure_encryption_settings": {"azure_tenant_id"}, + "resources.cluster_policies.*.libraries[*].cran": {"package"}, + "resources.cluster_policies.*.libraries[*].maven": {"coordinates"}, + "resources.cluster_policies.*.libraries[*].pypi": {"package"}, + "resources.clusters.*.cluster_log_conf.dbfs": {"destination"}, "resources.clusters.*.cluster_log_conf.s3": {"destination"}, "resources.clusters.*.cluster_log_conf.volumes": {"destination"}, From 46bcd67ae5a9cff247754ecdcb7841274399f365 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:34:59 +0000 Subject: [PATCH 18/44] fix workspace_open tests for cluster_policies resource type Adding cluster_policies with a workspace URL pattern extended the list of openable resource types, but the workspace_open command tests hardcoded the old list. Add cluster_policies to the expected completion, help text, and unknown-type error assertions. Co-authored-by: Isaac --- cmd/experimental/workspace_open_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 71904677781..95502694472 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -110,6 +110,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "alerts", "apps", "catalogs", + "cluster_policies", "clusters", "dashboards", "database_catalogs", @@ -148,7 +149,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") From ee55a1524ff76a072ef03953981d98caca79ee71 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:40:52 +0000 Subject: [PATCH 19/44] update experimental/open acceptance golden for cluster_policies The workspace open command's supported-resource-type list now includes cluster_policies; regenerate the golden output. Co-authored-by: Isaac --- acceptance/experimental/open/output.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index 1cd89cceda4..75591ed7423 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,13 +9,14 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , alerts apps catalogs +cluster_policies clusters dashboards database_catalogs From 98576cda6ad0168c76b8ae7929f83c32b283b3e5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 14:50:37 +0000 Subject: [PATCH 20/44] add cluster_policies invariant test coverage TestInvariantConfigsCoverage requires every resource type to have an invariant config. Add a cluster_policy config and register it in the invariant matrix. Exclude it from the migrate suite (terraform-seeded; the resource is direct-only) and continue_293 (unsupported on the old CLI). Regenerate the affected out.test.toml snapshots. Co-authored-by: Isaac --- .../bundle/invariant/configs/cluster_policy.yml.tmpl | 8 ++++++++ acceptance/bundle/invariant/continue_293/test.toml | 3 +++ .../bundle/invariant/delete_idempotent/out.test.toml | 1 + .../bundle/invariant/destroy_idempotent/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 2 ++ acceptance/bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + 7 files changed, 17 insertions(+) create mode 100644 acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl new file mode 100644 index 00000000000..aa514b0e0db --- /dev/null +++ b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + cluster_policies: + foo: + name: test-cluster-policy-$UNIQUE_NAME + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 1289c80af5e..f174ba144c4 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -16,6 +16,9 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # instance_pools resource is not supported on v0.293.0 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] +# cluster_policies resource is not supported on v0.293.0 +EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] + # job_runs resource is not supported on v0.293.0 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index cb927f4a69a..aa24bf58eed 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -14,6 +14,8 @@ EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmp EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # Instance pools are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] +# Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index faa2872a3b0..5c63aeef5b6 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", From 3aac44d084cc0b1e24812f9a1ffdbd9581ff26b5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 12:11:21 +0000 Subject: [PATCH 21/44] test: expand cluster_policies acceptance coverage Restructure the single cluster_policies acceptance test into a group: - move the existing test unchanged into basic/ - add job_ref/: a job task referencing the policy via ${resources.cluster_policies.pol.id}, asserting the direct engine orders policy create before job create (and job delete before policy delete on destroy) and resolves the policy id into the job body - add definition_multiline/: a block-scalar JSON definition, asserting it is preserved as a newline-escaped string end to end No production code change. Co-authored-by: Isaac --- .../{ => basic}/databricks.yml | 0 .../{ => basic}/out.test.toml | 0 .../cluster_policies/{ => basic}/output.txt | 0 .../cluster_policies/{ => basic}/script | 0 .../definition_multiline/databricks.yml | 14 ++ .../definition_multiline/out.test.toml | 2 + .../definition_multiline/output.txt | 34 +++++ .../definition_multiline/script | 9 ++ .../cluster_policies/job_ref/databricks.yml | 18 +++ .../cluster_policies/job_ref/out.test.toml | 2 + .../cluster_policies/job_ref/output.txt | 123 ++++++++++++++++++ .../resources/cluster_policies/job_ref/script | 17 +++ 12 files changed, 219 insertions(+) rename acceptance/bundle/resources/cluster_policies/{ => basic}/databricks.yml (100%) rename acceptance/bundle/resources/cluster_policies/{ => basic}/out.test.toml (100%) rename acceptance/bundle/resources/cluster_policies/{ => basic}/output.txt (100%) rename acceptance/bundle/resources/cluster_policies/{ => basic}/script (100%) create mode 100644 acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/definition_multiline/script create mode 100644 acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/job_ref/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/job_ref/script diff --git a/acceptance/bundle/resources/cluster_policies/databricks.yml b/acceptance/bundle/resources/cluster_policies/basic/databricks.yml similarity index 100% rename from acceptance/bundle/resources/cluster_policies/databricks.yml rename to acceptance/bundle/resources/cluster_policies/basic/databricks.yml diff --git a/acceptance/bundle/resources/cluster_policies/out.test.toml b/acceptance/bundle/resources/cluster_policies/basic/out.test.toml similarity index 100% rename from acceptance/bundle/resources/cluster_policies/out.test.toml rename to acceptance/bundle/resources/cluster_policies/basic/out.test.toml diff --git a/acceptance/bundle/resources/cluster_policies/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt similarity index 100% rename from acceptance/bundle/resources/cluster_policies/output.txt rename to acceptance/bundle/resources/cluster_policies/basic/output.txt diff --git a/acceptance/bundle/resources/cluster_policies/script b/acceptance/bundle/resources/cluster_policies/basic/script similarity index 100% rename from acceptance/bundle/resources/cluster_policies/script rename to acceptance/bundle/resources/cluster_policies/basic/script diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml new file mode 100644 index 00000000000..395e21aa5d0 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_definition_multiline + +resources: + cluster_policies: + pol: + name: my_policy + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt new file mode 100644 index 00000000000..e6912527df8 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle validate -o json +{ + "pol": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Create body preserves the block-scalar definition as a newline-escaped string +>>> print_requests.py //policies/clusters +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/script b/acceptance/bundle/resources/cluster_policies/definition_multiline/script new file mode 100644 index 00000000000..c134016b6e7 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle deploy + +title "Create body preserves the block-scalar definition as a newline-escaped string" +trace print_requests.py //policies/clusters + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml new file mode 100644 index 00000000000..1eb611c2a73 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: cluster_policy_job_ref + +resources: + cluster_policies: + pol: + name: my_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + + jobs: + j: + name: my_job + tasks: + - task_key: main + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml b/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt new file mode 100644 index 00000000000..bdf8a37c6bc --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -0,0 +1,123 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my_job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } +} + +=== Update both the policy and the job +>>> update_file.py databricks.yml my_policy my_policy_2 + +>>> update_file.py databricks.yml my_job my_job_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy_2", + "policy_id": "[POL_ID]" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [NUMID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my_job_2", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } + } +} + +=== Destroy: job delete precedes policy delete +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/delete", + "body": { + "policy_id": "[POL_ID]" + } +} diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/script b/acceptance/bundle/resources/cluster_policies/job_ref/script new file mode 100644 index 00000000000..bd4b09777bb --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/script @@ -0,0 +1,17 @@ +trace $CLI bundle deploy + +# Register [POL_ID] so the resolved reference in the job body is deterministic. +pol_id=`read_id.py pol` + +title "Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id" +trace print_requests.py //policies/clusters //jobs + +title "Update both the policy and the job" +trace update_file.py databricks.yml my_policy my_policy_2 +trace update_file.py databricks.yml my_job my_job_2 +trace $CLI bundle deploy +trace print_requests.py //policies/clusters //jobs + +title "Destroy: job delete precedes policy delete" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //policies/clusters //jobs From ffdd6df422c34a2cd0bc5ea15ff8f0e33f50334a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 12:14:59 +0000 Subject: [PATCH 22/44] Support authoring cluster_policies definition as inline YAML The cluster policy `definition` was a plain JSON string. Add a top-level `Definition any` field that shadows the embedded compute.CreatePolicy string so the definition can also be written as native YAML. ConfigureClusterPolicyDefinition normalizes an inline map/sequence to a JSON string at the dyn layer (same approach as genie serialized_space), avoiding int/float structdiff drift; PrepareState copies the normalized string into state. A string definition passes through unchanged. Co-authored-by: Isaac --- .../configure_cluster_policy_definition.go | 65 +++++++++++++++++++ .../resourcemutator/resource_mutator.go | 4 ++ bundle/config/resources/cluster_policy.go | 5 ++ bundle/direct/dresources/cluster_policy.go | 8 ++- bundle/schema/jsonschema.json | 2 +- 5 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go new file mode 100644 index 00000000000..093abb1d330 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go @@ -0,0 +1,65 @@ +package resourcemutator + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const definitionFieldName = "definition" + +type configureClusterPolicyDefinition struct{} + +func ConfigureClusterPolicyDefinition() bundle.Mutator { + return &configureClusterPolicyDefinition{} +} + +func (c configureClusterPolicyDefinition) Name() string { + return "ConfigureClusterPolicyDefinition" +} + +func (c configureClusterPolicyDefinition) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("cluster_policies"), + dyn.AnyKey(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + def := v.Get(definitionFieldName) + + // Marshal an inline structured definition to a JSON string so both + // config-side and state-side carry the same plain string. Otherwise + // YAML decodes small ints as Go `int` while state JSON round-trip + // decodes them as `float64`, and structdiff reports false drift. + switch def.Kind() { + case dyn.KindInvalid, dyn.KindNil, dyn.KindString: + // KindInvalid means definition is absent; leave it for backend validation. + return v, nil + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(def.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline definition: %w", err) + } + return dyn.Set(v, definitionFieldName, dyn.V(string(jsonBytes))) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("definition must be a string, map, or sequence, got %s", def.Kind()), + Locations: def.Locations(), + }) + return v, nil + } + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e8c33f0c59b..d771c0b3f57 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -208,6 +208,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Updates (dynamic): resources.genie_spaces.*.serialized_space ConfigureGenieSpaceSerializedSpace(), + // Reads (dynamic): resources.cluster_policies.*.definition + // Updates (dynamic): resources.cluster_policies.*.definition (inline YAML -> JSON string) + ConfigureClusterPolicyDefinition(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/resources/cluster_policy.go b/bundle/config/resources/cluster_policy.go index cc636fe2f58..98263d1a676 100644 --- a/bundle/config/resources/cluster_policy.go +++ b/bundle/config/resources/cluster_policy.go @@ -14,6 +14,11 @@ import ( type ClusterPolicy struct { BaseResource compute.CreatePolicy + + // Shadows the embedded compute.CreatePolicy.Definition (a string). `any` lets the + // definition be authored as inline YAML; ConfigureClusterPolicyDefinition normalizes + // it to a JSON string before deploy. + Definition any `json:"definition,omitempty"` } func (s *ClusterPolicy) UnmarshalJSON(b []byte) error { diff --git a/bundle/direct/dresources/cluster_policy.go b/bundle/direct/dresources/cluster_policy.go index f6c0db22569..e422a227616 100644 --- a/bundle/direct/dresources/cluster_policy.go +++ b/bundle/direct/dresources/cluster_policy.go @@ -18,7 +18,13 @@ func (*ResourceClusterPolicy) New(client *databricks.WorkspaceClient) *ResourceC } func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *compute.CreatePolicy { - return &input.CreatePolicy + cp := input.CreatePolicy + // The top-level Definition shadows the embedded string; ConfigureClusterPolicyDefinition + // has already normalized it to a JSON string by this point. + if s, ok := input.Definition.(string); ok { + cp.Definition = s + } + return &cp } // RemapState copies the config fields shared by Policy and CreatePolicy; diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index e136dab10eb..85de80de072 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -596,7 +596,7 @@ "properties": { "definition": { "description": "Policy definition document expressed in [Databricks Cluster Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).", - "$ref": "#/$defs/string" + "$ref": "#/$defs/interface" }, "description": { "description": "Additional human-readable description of the cluster policy.", From 43698b522a33dd43a7f7983dae2e04f5b1b1f0c0 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 12:17:38 +0000 Subject: [PATCH 23/44] test: cover cluster_policies inline-YAML definition - unit test for ConfigureClusterPolicyDefinition (string passthrough, map/sequence -> JSON string, invalid-kind diagnostic, absent field) - unit test for ResourceClusterPolicy.PrepareState copying the normalized string into state - acceptance test definition_yaml/: a native YAML mapping serializes to the same compact JSON string the API receives as the basic test Co-authored-by: Isaac --- .../definition_yaml/databricks.yml | 11 +++ .../definition_yaml/out.test.toml | 2 + .../definition_yaml/output.txt | 34 +++++++++ .../cluster_policies/definition_yaml/script | 9 +++ ...onfigure_cluster_policy_definition_test.go | 74 +++++++++++++++++++ .../direct/dresources/cluster_policy_test.go | 47 ++++++++++++ 6 files changed, 177 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/definition_yaml/script create mode 100644 bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go create mode 100644 bundle/direct/dresources/cluster_policy_test.go diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml new file mode 100644 index 00000000000..2b5685d1c84 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: cluster_policy_definition_yaml + +resources: + cluster_policies: + pol: + name: my_policy + definition: + spark_version: + type: fixed + value: 13.3.x-scala2.12 diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt new file mode 100644 index 00000000000..ac4229558e4 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle validate -o json +{ + "pol": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Native YAML definition serializes to the compact JSON string the API receives +>>> print_requests.py //policies/clusters +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/script b/acceptance/bundle/resources/cluster_policies/definition_yaml/script new file mode 100644 index 00000000000..d4579c26c5d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle deploy + +title "Native YAML definition serializes to the compact JSON string the API receives" +trace print_requests.py //policies/clusters + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go new file mode 100644 index 00000000000..ad909c6a47b --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go @@ -0,0 +1,74 @@ +package resourcemutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigureClusterPolicyDefinition(t *testing.T) { + tests := []struct { + name string + definition any + wantDefinition any + // wantErr, when non-empty, is a substring expected in the diagnostics. + wantErr string + }{ + { + // Inline maps are marshaled to a compact JSON string with sorted keys + // so config and state hold an identical string and don't drift. + name: "inline map is marshaled to a JSON string", + definition: map[string]any{"spark_version": map[string]any{"type": "fixed", "value": "13.3.x"}}, + wantDefinition: `{"spark_version":{"type":"fixed","value":"13.3.x"}}`, + }, + { + name: "inline sequence is marshaled to a JSON string", + definition: []any{"a", "b"}, + wantDefinition: `["a","b"]`, + }, + { + name: "inline string is left unchanged", + definition: `{"spark_version":{"type":"fixed"}}`, + wantDefinition: `{"spark_version":{"type":"fixed"}}`, + }, + { + name: "absent definition passes through", + wantDefinition: nil, + }, + { + name: "non-structured definition is rejected", + definition: true, + wantErr: "definition must be a string, map, or sequence, got bool", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cp := &resources.ClusterPolicy{Definition: tt.definition} + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + ClusterPolicies: map[string]*resources.ClusterPolicy{"pol": cp}, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, resourcemutator.ConfigureClusterPolicyDefinition()) + + if tt.wantErr != "" { + require.Error(t, diags.Error()) + assert.ErrorContains(t, diags.Error(), tt.wantErr) + return + } + + require.NoError(t, diags.Error()) + assert.Equal(t, tt.wantDefinition, b.Config.Resources.ClusterPolicies["pol"].Definition) + }) + } +} diff --git a/bundle/direct/dresources/cluster_policy_test.go b/bundle/direct/dresources/cluster_policy_test.go new file mode 100644 index 00000000000..60377bfd0b3 --- /dev/null +++ b/bundle/direct/dresources/cluster_policy_test.go @@ -0,0 +1,47 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" +) + +func TestClusterPolicyPrepareState(t *testing.T) { + tests := []struct { + name string + definition any + want string + }{ + { + // The normal post-mutator case: definition is already a JSON string. + name: "string definition is copied into state", + definition: `{"spark_version":{"type":"fixed"}}`, + want: `{"spark_version":{"type":"fixed"}}`, + }, + { + // ConfigureClusterPolicyDefinition guarantees a string, so a non-string + // is ignored rather than reaching the API. + name: "non-string definition is ignored", + definition: map[string]any{"spark_version": "fixed"}, + want: "", + }, + { + name: "absent definition leaves state empty", + definition: nil, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := &resources.ClusterPolicy{Definition: tt.definition} + input.Name = "my_policy" + + got := (*ResourceClusterPolicy)(nil).PrepareState(input) + + assert.Equal(t, tt.want, got.Definition) + assert.Equal(t, "my_policy", got.Name) + }) + } +} From 671f0e2ced95462f3f16c06e75daff315aa9e00d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 12:25:36 +0000 Subject: [PATCH 24/44] regenerate refschema for cluster_policies inline-YAML definition The inline-YAML definition feature added an 'any'-typed definition input field but did not regenerate out.fields.txt, failing validate-generated. Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index a807286ffe0..ac9a5e87d5e 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -281,6 +281,7 @@ resources.catalogs.*.grants[*].privileges []catalog.Privilege ALL resources.catalogs.*.grants[*].privileges[*] catalog.Privilege ALL resources.cluster_policies.*.created_at_timestamp int64 REMOTE resources.cluster_policies.*.creator_user_name string REMOTE +resources.cluster_policies.*.definition any INPUT resources.cluster_policies.*.definition string ALL resources.cluster_policies.*.description string ALL resources.cluster_policies.*.id string INPUT From e5280fd15c88f686105e0e1e2a81ac722369104c Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 13:10:31 +0000 Subject: [PATCH 25/44] test: cluster_policies rejected in terraform mode Add a direct-only acceptance test asserting that deploying a cluster_policies resource with DATABRICKS_BUNDLE_ENGINE=terraform fails with an actionable error, mirroring secrets/direct-only. Co-authored-by: Isaac --- .../cluster_policies/direct-only/databricks.yml | 8 ++++++++ .../cluster_policies/direct-only/out.test.toml | 2 ++ .../resources/cluster_policies/direct-only/output.txt | 11 +++++++++++ .../resources/cluster_policies/direct-only/script | 4 ++++ .../resources/cluster_policies/direct-only/test.toml | 5 +++++ 5 files changed, 30 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/script create mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml b/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml new file mode 100644 index 00000000000..5c3ed8abd0b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: cluster_policy_direct_only + +resources: + cluster_policies: + pol: + name: my_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml b/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml new file mode 100644 index 00000000000..d2059b4b5d7 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/output.txt b/acceptance/bundle/resources/cluster_policies/direct-only/output.txt new file mode 100644 index 00000000000..8656fb8e6f9 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/output.txt @@ -0,0 +1,11 @@ + +=== Deploy should fail in terraform mode +>>> [CLI] bundle deploy +Error: Cluster Policy resources are only supported with direct deployment mode + in databricks.yml:6:5 + +Cluster Policy resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use cluster_policy resources. +Learn more at https://docs.databricks.com/dev-tools/bundles/direct + + +Exit code: 1 diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/script b/acceptance/bundle/resources/cluster_policies/direct-only/script new file mode 100644 index 00000000000..db1c9b194ba --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/script @@ -0,0 +1,4 @@ +title "Deploy should fail in terraform mode" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Cluster Policy resources are only supported with direct deployment mode" \ + "DATABRICKS_BUNDLE_ENGINE" diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/test.toml b/acceptance/bundle/resources/cluster_policies/direct-only/test.toml new file mode 100644 index 00000000000..554b3c0b60d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/test.toml @@ -0,0 +1,5 @@ +Cloud = false +RecordRequests = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] + +Ignore = [".databricks"] From be7ccb25f49b6c341e623e5184f69e30882cd0a3 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 13:12:31 +0000 Subject: [PATCH 26/44] test: cluster_policies dangling reference fails to plan Add an acceptance test where a job new_cluster references an undeclared cluster_policies resource; bundle plan fails with a config-time dependency-resolution error. Co-authored-by: Isaac --- .../cluster_policies/missing_ref/databricks.yml | 14 ++++++++++++++ .../cluster_policies/missing_ref/out.plan.txt | 2 ++ .../cluster_policies/missing_ref/out.test.toml | 2 ++ .../cluster_policies/missing_ref/output.txt | 2 ++ .../resources/cluster_policies/missing_ref/script | 2 ++ .../cluster_policies/missing_ref/test.toml | 1 + 6 files changed, 23 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt create mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/script create mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml b/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml new file mode 100644 index 00000000000..55419f57f30 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_missing_ref + +resources: + jobs: + j: + name: my_job + tasks: + - task_key: main + new_cluster: + # References a cluster policy that is not declared in this bundle. + policy_id: ${resources.cluster_policies.missing.id} + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt b/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt new file mode 100644 index 00000000000..e43b42e0e8f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt @@ -0,0 +1,2 @@ +Error: invalid dependency "${resources.cluster_policies.missing.id}", no such node "resources.cluster_policies.missing" + diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml b/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt b/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt new file mode 100644 index 00000000000..d06d4ddc5ef --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt @@ -0,0 +1,2 @@ + +=== Plan fails: job references an undeclared cluster policy \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/script b/acceptance/bundle/resources/cluster_policies/missing_ref/script new file mode 100644 index 00000000000..2fa06ac7476 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/script @@ -0,0 +1,2 @@ +title "Plan fails: job references an undeclared cluster policy" +musterr $CLI bundle plan &> out.plan.txt diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml b/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml @@ -0,0 +1 @@ +RecordRequests = false From e9aa452e7ff5ea8258508eb6a028b293ad16e6c6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 13:19:18 +0000 Subject: [PATCH 27/44] test: cross-bundle cluster_policy use via hardcoded id One bundle creates a cluster policy; a second, separate bundle consumes it by hardcoding the generated policy_id (captured from the first bundle's state) in a job's new_cluster. Asserts the consumer job's create request carries the producer's policy id. Co-authored-by: Isaac --- .../cross_bundle_id/bundle_a/databricks.yml | 8 ++ .../cross_bundle_id/bundle_b/databricks.yml | 14 ++++ .../cross_bundle_id/out.test.toml | 2 + .../cross_bundle_id/output.txt | 73 +++++++++++++++++++ .../cluster_policies/cross_bundle_id/script | 17 +++++ .../cross_bundle_id/test.toml | 1 + 6 files changed, 115 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/script create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml new file mode 100644 index 00000000000..195681f9c74 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: cluster_policy_producer + +resources: + cluster_policies: + pol: + name: shared_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml new file mode 100644 index 00000000000..afacb0e118e --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_consumer + +resources: + jobs: + j: + name: consumer_job + tasks: + - task_key: main + new_cluster: + # Replaced at test time with the policy id created by bundle_a. + policy_id: PLACEHOLDER_POLICY_ID + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt new file mode 100644 index 00000000000..84da21ee84c --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt @@ -0,0 +1,73 @@ + +=== Bundle A creates the cluster policy +>>> withdir bundle_a [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Inject A's policy id into bundle B, then deploy B +>>> update_file.py bundle_b/databricks.yml PLACEHOLDER_POLICY_ID [POL_ID] + +>>> withdir bundle_b [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Policy created by A, then B's job carries that same policy id +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "shared_policy" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "consumer_job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } +} + +=== Cleanup +>>> withdir bundle_b [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default + +Deleting files... +Destroy complete! + +>>> withdir bundle_a [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script new file mode 100644 index 00000000000..e2b47428da7 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script @@ -0,0 +1,17 @@ +title "Bundle A creates the cluster policy" +trace withdir bundle_a $CLI bundle deploy + +# Capture A's server-generated policy id and register [POL_ID]. +pol_id=$(withdir bundle_a read_id.py pol) + +title "Inject A's policy id into bundle B, then deploy B" +trace update_file.py bundle_b/databricks.yml PLACEHOLDER_POLICY_ID "$pol_id" +trace withdir bundle_b $CLI bundle deploy + +title "Policy created by A, then B's job carries that same policy id" +trace print_requests.py //policies/clusters //jobs + +title "Cleanup" +trace withdir bundle_b $CLI bundle destroy --auto-approve +trace withdir bundle_a $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml new file mode 100644 index 00000000000..601384fdf96 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks"] From 60ede9e994f4b2e801ef2f7ca8fd18757c2591f0 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 13:38:18 +0000 Subject: [PATCH 28/44] test: cross-bundle cluster_policy use via lookup by name Make the testserver's /api/2.0/policies/clusters/list stateful so it returns policies created via the create handler, seeding the two legacy names the variable-lookup tests rely on. Add an acceptance test where a consumer bundle resolves a policy created by a separate producer bundle through a variable lookup {cluster_policy: }, then uses the resolved id in a job. Co-authored-by: Isaac --- .../bundle_a/databricks.yml | 8 ++ .../bundle_b/databricks.yml | 19 +++++ .../cross_bundle_lookup/out.test.toml | 2 + .../cross_bundle_lookup/output.txt | 74 +++++++++++++++++++ .../cross_bundle_lookup/script | 17 +++++ .../cross_bundle_lookup/test.toml | 1 + libs/testserver/cluster_policies.go | 18 +++++ libs/testserver/fake_workspace.go | 9 ++- libs/testserver/handlers.go | 13 +--- 9 files changed, 147 insertions(+), 14 deletions(-) create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script create mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml new file mode 100644 index 00000000000..fd1856beb60 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: cluster_policy_producer + +resources: + cluster_policies: + pol: + name: shared_lookup_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml new file mode 100644 index 00000000000..0789f610f03 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: cluster_policy_consumer + +variables: + policy: + description: Resolve the policy created by bundle_a by name. + lookup: + cluster_policy: shared_lookup_policy + +resources: + jobs: + j: + name: consumer_job + tasks: + - task_key: main + new_cluster: + policy_id: ${var.policy} + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt new file mode 100644 index 00000000000..b5061bbc3f3 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt @@ -0,0 +1,74 @@ + +=== Bundle A creates the cluster policy +>>> withdir bundle_a [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Bundle B resolves the policy by name via lookup +>>> withdir bundle_b [CLI] bundle validate -o json +{ + "policy": { + "description": "Resolve the policy created by bundle_a by name.", + "lookup": { + "cluster_policy": "shared_lookup_policy" + }, + "value": "[POL_ID]" + } +} + +=== Deploy B; its job carries the resolved policy id +>>> withdir bundle_b [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "consumer_job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } +} + +=== Cleanup +>>> withdir bundle_b [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default + +Deleting files... +Destroy complete! + +>>> withdir bundle_a [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script new file mode 100644 index 00000000000..affac0bb5bb --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script @@ -0,0 +1,17 @@ +title "Bundle A creates the cluster policy" +trace withdir bundle_a $CLI bundle deploy + +# Register [POL_ID] for A's server-generated policy id. +pol_id=$(withdir bundle_a read_id.py pol) + +title "Bundle B resolves the policy by name via lookup" +trace withdir bundle_b $CLI bundle validate -o json | jq '.variables' + +title "Deploy B; its job carries the resolved policy id" +trace withdir bundle_b $CLI bundle deploy +trace print_requests.py //jobs + +title "Cleanup" +trace withdir bundle_b $CLI bundle destroy --auto-approve +trace withdir bundle_a $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml new file mode 100644 index 00000000000..601384fdf96 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks"] diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index 61c69347baf..5616a32a00f 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -3,6 +3,7 @@ package testserver import ( "encoding/json" "fmt" + "slices" "github.com/databricks/databricks-sdk-go/service/compute" ) @@ -24,6 +25,23 @@ func (s *FakeWorkspace) ClusterPoliciesCreate(req Request) any { return Response{Body: compute.CreatePolicyResponse{PolicyId: id}} } +func (s *FakeWorkspace) ClusterPoliciesList(req Request) any { + defer s.LockUnlock()() + + ids := make([]string, 0, len(s.ClusterPolicies)) + for id := range s.ClusterPolicies { + ids = append(ids, id) + } + slices.Sort(ids) + + policies := make([]compute.Policy, 0, len(ids)) + for _, id := range ids { + policies = append(policies, s.ClusterPolicies[id]) + } + + return Response{Body: compute.ListPoliciesResponse{Policies: policies}} +} + func (s *FakeWorkspace) ClusterPoliciesGet(req Request, policyId string) any { defer s.LockUnlock()() diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 9dbca5373e5..1982e13658a 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -429,8 +429,13 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, - InstancePools: map[string]compute.GetInstancePool{}, - ClusterPolicies: map[string]compute.Policy{}, + InstancePools: map[string]compute.GetInstancePool{}, + ClusterPolicies: map[string]compute.Policy{ + // Seeded so the stateful list keeps backing the variable-lookup tests + // (e.g. acceptance/bundle/variables/env_overrides resolves these by name). + "5678": {PolicyId: "5678", Name: "wrong-cluster-policy"}, + "9876": {PolicyId: "9876", Name: "some-test-cluster-policy"}, + }, VectorSearchIndexesPendingDeletion: map[string]int{}, } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 282dbec9338..05ac9df9370 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -27,18 +27,7 @@ var TestMetastore = catalog.MetastoreAssignment{ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/policies/clusters/list", func(req Request) any { - return compute.ListPoliciesResponse{ - Policies: []compute.Policy{ - { - PolicyId: "5678", - Name: "wrong-cluster-policy", - }, - { - PolicyId: "9876", - Name: "some-test-cluster-policy", - }, - }, - } + return req.Workspace.ClusterPoliciesList(req) }) server.Handle("GET", "/api/2.0/instance-pools/list", func(req Request) any { From 780cbb8680610531829ef37f9c6d5f84733891cc Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 13 Aug 2026 13:48:03 +0000 Subject: [PATCH 29/44] test: vary cluster_policies definition authoring across fixtures Instead of inline JSON everywhere, spread the three authoring forms across the non-targeted tests for incidental coverage: job_ref and cross_bundle_lookup use a multiline JSON block scalar, direct-only uses native YAML, cross_bundle_id keeps inline JSON. basic and the two targeted definition tests are unchanged. Co-authored-by: Isaac --- .../cross_bundle_lookup/bundle_a/databricks.yml | 8 +++++++- .../resources/cluster_policies/direct-only/databricks.yml | 5 ++++- .../resources/cluster_policies/job_ref/databricks.yml | 8 +++++++- .../bundle/resources/cluster_policies/job_ref/output.txt | 4 ++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml index fd1856beb60..86501c512c3 100644 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml @@ -5,4 +5,10 @@ resources: cluster_policies: pol: name: shared_lookup_policy - definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml b/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml index 5c3ed8abd0b..8b716db56df 100644 --- a/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml @@ -5,4 +5,7 @@ resources: cluster_policies: pol: name: my_policy - definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + definition: + spark_version: + type: fixed + value: 13.3.x-scala2.12 diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml index 1eb611c2a73..7bc49b780c2 100644 --- a/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml @@ -5,7 +5,13 @@ resources: cluster_policies: pol: name: my_policy - definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } jobs: j: diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt index bdf8a37c6bc..00620492ca3 100644 --- a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -11,7 +11,7 @@ Deployment complete! "method": "POST", "path": "/api/2.0/policies/clusters/create", "body": { - "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", "name": "my_policy" } } @@ -59,7 +59,7 @@ Deployment complete! "method": "POST", "path": "/api/2.0/policies/clusters/edit", "body": { - "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", "name": "my_policy_2", "policy_id": "[POL_ID]" } From 2df7f3c626ad1dc63b9cad5dd41c833c9cc8f8c2 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 09:59:46 +0000 Subject: [PATCH 30/44] add a test for an out of band change --- .../out_of_band_change/databricks.yml | 8 +++ .../out_of_band_change/out.test.toml | 2 + .../out_of_band_change/output.txt | 51 +++++++++++++++++++ .../out_of_band_change/script | 34 +++++++++++++ .../out_of_band_change/test.toml | 4 ++ 5 files changed, 99 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/out_of_band_change/script create mode 100644 acceptance/bundle/resources/cluster_policies/out_of_band_change/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml new file mode 100644 index 00000000000..5156a2b9b0f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test_cluster_policy + +resources: + cluster_policies: + test_cluster_policy: + name: my_cluster_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt new file mode 100644 index 00000000000..26c36bac429 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt @@ -0,0 +1,51 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op immediately after deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Edit the policy definition out of band +>>> [CLI] cluster-policies edit --json {"policy_id":"[TEST_CLUSTER_POLICY_ID]","name":"my_cluster_policy","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"} + +=== Plan detects the drift +>>> [CLI] bundle plan +update cluster_policies.test_cluster_policy + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +=== Redeploy reconciles the policy back to the configured definition +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the edit request restored the configured definition +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy", + "policy_id": "[TEST_CLUSTER_POLICY_ID]" + } +} + +=== Plan is a no-op again +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.test_cluster_policy + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script new file mode 100644 index 00000000000..47d8585c72d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script @@ -0,0 +1,34 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Plan is a no-op immediately after deploy" +trace $CLI bundle plan + +policy_id="$(read_id.py test_cluster_policy)" + +# Simulate an out-of-band change: edit the policy definition directly through the +# API, the way an admin would in the UI, without touching databricks.yml. The +# recorded bundle state is now stale, so the next plan must detect the drift. +title "Edit the policy definition out of band" +trace $CLI cluster-policies edit --json '{"policy_id":"'"$policy_id"'","name":"my_cluster_policy","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"}' + +# Discard the out-of-band request so the verification below captures only the +# reconciling edit issued by the redeploy. +rm -f out.requests.txt + +title "Plan detects the drift" +trace $CLI bundle plan + +title "Redeploy reconciles the policy back to the configured definition" +trace $CLI bundle deploy + +title "Verify the edit request restored the configured definition" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt + +title "Plan is a no-op again" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/test.toml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/test.toml new file mode 100644 index 00000000000..b03f24bc233 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/test.toml @@ -0,0 +1,4 @@ +Ignore = [ + ".databricks", + "databricks.yml", +] From bfacfa3a6a181ad7b6ca61892f32e46df5123a64 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 10:44:16 +0000 Subject: [PATCH 31/44] add non string field to yaml --- .../cluster_policies/definition_yaml/databricks.yml | 7 +++++++ .../resources/cluster_policies/definition_yaml/output.txt | 6 +++--- .../resources/cluster_policies/definition_yaml/script | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml index 2b5685d1c84..49ecc474a22 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml @@ -9,3 +9,10 @@ resources: spark_version: type: fixed value: 13.3.x-scala2.12 + autotermination_minutes: + type: range + minValue: 10 + maxValue: 120 + enable_elastic_disk: + type: fixed + value: true diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt index ac4229558e4..edc38996d16 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -2,7 +2,7 @@ >>> [CLI] bundle validate -o json { "pol": { - "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", "name": "my_policy" } } @@ -13,13 +13,13 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Native YAML definition serializes to the compact JSON string the API receives +=== Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings) >>> print_requests.py //policies/clusters { "method": "POST", "path": "/api/2.0/policies/clusters/create", "body": { - "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", "name": "my_policy" } } diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/script b/acceptance/bundle/resources/cluster_policies/definition_yaml/script index d4579c26c5d..083c796fd5d 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/script +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/script @@ -2,7 +2,7 @@ trace $CLI bundle validate -o json | jq ".resources.cluster_policies" trace $CLI bundle deploy -title "Native YAML definition serializes to the compact JSON string the API receives" +title "Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings)" trace print_requests.py //policies/clusters trace $CLI bundle destroy --auto-approve From d9f0ca0450b70814f7cb20d1aa2a944f6e7e5120 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 10:47:30 +0000 Subject: [PATCH 32/44] delete tests not specific to cluster policies --- .../cluster_policies/direct-only/databricks.yml | 11 ----------- .../cluster_policies/direct-only/out.test.toml | 2 -- .../cluster_policies/direct-only/output.txt | 11 ----------- .../resources/cluster_policies/direct-only/script | 4 ---- .../cluster_policies/direct-only/test.toml | 5 ----- .../cluster_policies/missing_ref/databricks.yml | 14 -------------- .../cluster_policies/missing_ref/out.plan.txt | 2 -- .../cluster_policies/missing_ref/out.test.toml | 2 -- .../cluster_policies/missing_ref/output.txt | 2 -- .../resources/cluster_policies/missing_ref/script | 2 -- .../cluster_policies/missing_ref/test.toml | 1 - 11 files changed, 56 deletions(-) delete mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml delete mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/output.txt delete mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/script delete mode 100644 acceptance/bundle/resources/cluster_policies/direct-only/test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml delete mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt delete mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/output.txt delete mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/script delete mode 100644 acceptance/bundle/resources/cluster_policies/missing_ref/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml b/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml deleted file mode 100644 index 8b716db56df..00000000000 --- a/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: cluster_policy_direct_only - -resources: - cluster_policies: - pol: - name: my_policy - definition: - spark_version: - type: fixed - value: 13.3.x-scala2.12 diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml b/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml deleted file mode 100644 index d2059b4b5d7..00000000000 --- a/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/output.txt b/acceptance/bundle/resources/cluster_policies/direct-only/output.txt deleted file mode 100644 index 8656fb8e6f9..00000000000 --- a/acceptance/bundle/resources/cluster_policies/direct-only/output.txt +++ /dev/null @@ -1,11 +0,0 @@ - -=== Deploy should fail in terraform mode ->>> [CLI] bundle deploy -Error: Cluster Policy resources are only supported with direct deployment mode - in databricks.yml:6:5 - -Cluster Policy resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use cluster_policy resources. -Learn more at https://docs.databricks.com/dev-tools/bundles/direct - - -Exit code: 1 diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/script b/acceptance/bundle/resources/cluster_policies/direct-only/script deleted file mode 100644 index db1c9b194ba..00000000000 --- a/acceptance/bundle/resources/cluster_policies/direct-only/script +++ /dev/null @@ -1,4 +0,0 @@ -title "Deploy should fail in terraform mode" -trace $CLI bundle deploy 2>&1 | contains.py \ - "Cluster Policy resources are only supported with direct deployment mode" \ - "DATABRICKS_BUNDLE_ENGINE" diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/test.toml b/acceptance/bundle/resources/cluster_policies/direct-only/test.toml deleted file mode 100644 index 554b3c0b60d..00000000000 --- a/acceptance/bundle/resources/cluster_policies/direct-only/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -Cloud = false -RecordRequests = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] - -Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml b/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml deleted file mode 100644 index 55419f57f30..00000000000 --- a/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml +++ /dev/null @@ -1,14 +0,0 @@ -bundle: - name: cluster_policy_missing_ref - -resources: - jobs: - j: - name: my_job - tasks: - - task_key: main - new_cluster: - # References a cluster policy that is not declared in this bundle. - policy_id: ${resources.cluster_policies.missing.id} - spark_version: 13.3.x-scala2.12 - num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt b/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt deleted file mode 100644 index e43b42e0e8f..00000000000 --- a/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt +++ /dev/null @@ -1,2 +0,0 @@ -Error: invalid dependency "${resources.cluster_policies.missing.id}", no such node "resources.cluster_policies.missing" - diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml b/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml deleted file mode 100644 index 0938e678987..00000000000 --- a/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt b/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt deleted file mode 100644 index d06d4ddc5ef..00000000000 --- a/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt +++ /dev/null @@ -1,2 +0,0 @@ - -=== Plan fails: job references an undeclared cluster policy \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/script b/acceptance/bundle/resources/cluster_policies/missing_ref/script deleted file mode 100644 index 2fa06ac7476..00000000000 --- a/acceptance/bundle/resources/cluster_policies/missing_ref/script +++ /dev/null @@ -1,2 +0,0 @@ -title "Plan fails: job references an undeclared cluster policy" -musterr $CLI bundle plan &> out.plan.txt diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml b/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml deleted file mode 100644 index a030353d571..00000000000 --- a/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml +++ /dev/null @@ -1 +0,0 @@ -RecordRequests = false From aef7b32553bd4a6f04c4892fa3235d2a8308e088 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 12:27:48 +0000 Subject: [PATCH 33/44] add policy_family_definition_overrides support --- .../policy_family_overrides/databricks.yml | 34 ++++++++ .../policy_family_overrides/out.test.toml | 2 + .../policy_family_overrides/output.txt | 66 ++++++++++++++++ .../policy_family_overrides/script | 11 +++ .../configure_cluster_policy_definition.go | 50 ++++++------ ...onfigure_cluster_policy_definition_test.go | 79 ++++++++++++++----- bundle/config/resources/cluster_policy.go | 4 + bundle/direct/dresources/cluster_policy.go | 6 +- bundle/schema/jsonschema.json | 2 +- 9 files changed, 208 insertions(+), 46 deletions(-) create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_overrides/script diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml new file mode 100644 index 00000000000..a50210d97a6 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml @@ -0,0 +1,34 @@ +bundle: + name: cluster_policy_overrides + +resources: + cluster_policies: + # Overrides as a compact JSON string. + json_string: + name: policy_json_string + policy_family_id: personal-vm + policy_family_definition_overrides: '{"autotermination_minutes":{"type":"fixed","value":30}}' + + # Overrides as a multiline block-scalar JSON string (preserved verbatim). + multiline: + name: policy_multiline + policy_family_id: personal-vm + policy_family_definition_overrides: |- + { + "autotermination_minutes": { + "type": "fixed", + "value": 30 + } + } + + # Overrides as native YAML (normalized to compact JSON, numbers/booleans kept as JSON types). + yaml: + name: policy_yaml + policy_family_id: personal-vm + policy_family_definition_overrides: + autotermination_minutes: + type: fixed + value: 30 + enable_elastic_disk: + type: fixed + value: true diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt new file mode 100644 index 00000000000..fbc09a64dc5 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt @@ -0,0 +1,66 @@ + +>>> [CLI] bundle validate -o json +{ + "json_string": { + "name": "policy_json_string", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30}}", + "policy_family_id": "personal-vm" + }, + "multiline": { + "name": "policy_multiline", + "policy_family_definition_overrides": "{\n \"autotermination_minutes\": {\n \"type\": \"fixed\",\n \"value\": 30\n }\n}", + "policy_family_id": "personal-vm" + }, + "yaml": { + "name": "policy_yaml", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true}}", + "policy_family_id": "personal-vm" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved) +>>> print_requests.py //policies/clusters/create --sort +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "name": "policy_json_string", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30}}", + "policy_family_id": "personal-vm" + } +} +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "name": "policy_multiline", + "policy_family_definition_overrides": "{\n \"autotermination_minutes\": {\n \"type\": \"fixed\",\n \"value\": 30\n }\n}", + "policy_family_id": "personal-vm" + } +} +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "name": "policy_yaml", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true}}", + "policy_family_id": "personal-vm" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.json_string + delete resources.cluster_policies.multiline + delete resources.cluster_policies.yaml + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script new file mode 100644 index 00000000000..fe1e5bd0507 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script @@ -0,0 +1,11 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT +trace $CLI bundle deploy + +title "Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved)" +trace print_requests.py //policies/clusters/create --sort diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go index 093abb1d330..c0b1b4b5be7 100644 --- a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go @@ -10,7 +10,8 @@ import ( "github.com/databricks/cli/libs/dyn" ) -const definitionFieldName = "definition" +// jsonPolicyFields are the JSON-policy fields normalized from inline YAML to a JSON string. +var jsonPolicyFields = []string{"definition", "policy_family_definition_overrides"} type configureClusterPolicyDefinition struct{} @@ -33,30 +34,35 @@ func (c configureClusterPolicyDefinition) Apply(_ context.Context, b *bundle.Bun err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - def := v.Get(definitionFieldName) + for _, field := range jsonPolicyFields { + def := v.Get(field) - // Marshal an inline structured definition to a JSON string so both - // config-side and state-side carry the same plain string. Otherwise - // YAML decodes small ints as Go `int` while state JSON round-trip - // decodes them as `float64`, and structdiff reports false drift. - switch def.Kind() { - case dyn.KindInvalid, dyn.KindNil, dyn.KindString: - // KindInvalid means definition is absent; leave it for backend validation. - return v, nil - case dyn.KindMap, dyn.KindSequence: - jsonBytes, err := json.Marshal(def.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline definition: %w", err) + // Marshal an inline structured value to a JSON string so both + // config-side and state-side carry the same plain string. Otherwise + // YAML decodes small ints as Go `int` while state JSON round-trip + // decodes them as `float64`, and structdiff reports false drift. + switch def.Kind() { + case dyn.KindInvalid, dyn.KindNil, dyn.KindString: + // KindInvalid means the field is absent; leave it for backend validation. + continue + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(def.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline %s: %w", field, err) + } + v, err = dyn.Set(v, field, dyn.V(string(jsonBytes))) + if err != nil { + return dyn.InvalidValue, err + } + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s must be a string, map, or sequence, got %s", field, def.Kind()), + Locations: def.Locations(), + }) } - return dyn.Set(v, definitionFieldName, dyn.V(string(jsonBytes))) - default: - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("definition must be a string, map, or sequence, got %s", def.Kind()), - Locations: def.Locations(), - }) - return v, nil } + return v, nil }) }) diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go index ad909c6a47b..5e9d53fed4a 100644 --- a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go @@ -13,43 +13,74 @@ import ( func TestConfigureClusterPolicyDefinition(t *testing.T) { tests := []struct { - name string - definition any - wantDefinition any + name string + field string // "definition" or "policy_family_definition_overrides" + value any + want any // wantErr, when non-empty, is a substring expected in the diagnostics. wantErr string }{ { - // Inline maps are marshaled to a compact JSON string with sorted keys - // so config and state hold an identical string and don't drift. - name: "inline map is marshaled to a JSON string", - definition: map[string]any{"spark_version": map[string]any{"type": "fixed", "value": "13.3.x"}}, - wantDefinition: `{"spark_version":{"type":"fixed","value":"13.3.x"}}`, + name: "definition: inline map is marshaled to a JSON string", + field: "definition", + value: map[string]any{"spark_version": map[string]any{"type": "fixed", "value": "13.3.x"}}, + want: `{"spark_version":{"type":"fixed","value":"13.3.x"}}`, }, { - name: "inline sequence is marshaled to a JSON string", - definition: []any{"a", "b"}, - wantDefinition: `["a","b"]`, + name: "definition: inline sequence is marshaled to a JSON string", + field: "definition", + value: []any{"a", "b"}, + want: `["a","b"]`, }, { - name: "inline string is left unchanged", - definition: `{"spark_version":{"type":"fixed"}}`, - wantDefinition: `{"spark_version":{"type":"fixed"}}`, + name: "definition: inline string is left unchanged", + field: "definition", + value: `{"spark_version":{"type":"fixed"}}`, + want: `{"spark_version":{"type":"fixed"}}`, }, { - name: "absent definition passes through", - wantDefinition: nil, + name: "definition: absent passes through", + field: "definition", + want: nil, }, { - name: "non-structured definition is rejected", - definition: true, - wantErr: "definition must be a string, map, or sequence, got bool", + name: "definition: non-structured is rejected", + field: "definition", + value: true, + wantErr: "definition must be a string, map, or sequence, got bool", + }, + { + // Number stays a JSON number (30, not "30"). + name: "overrides: inline map is marshaled to a JSON string", + field: "policy_family_definition_overrides", + value: map[string]any{"autotermination_minutes": map[string]any{"type": "fixed", "value": 30}}, + want: `{"autotermination_minutes":{"type":"fixed","value":30}}`, + }, + { + name: "overrides: inline string is left unchanged", + field: "policy_family_definition_overrides", + value: `{"autotermination_minutes":{"type":"fixed"}}`, + want: `{"autotermination_minutes":{"type":"fixed"}}`, + }, + { + name: "overrides: non-structured is rejected", + field: "policy_family_definition_overrides", + value: true, + wantErr: "policy_family_definition_overrides must be a string, map, or sequence, got bool", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cp := &resources.ClusterPolicy{Definition: tt.definition} + cp := &resources.ClusterPolicy{} + switch tt.field { + case "policy_family_definition_overrides": + cp.PolicyFamilyDefinitionOverrides = tt.value + case "definition": + cp.Definition = tt.value + default: + t.Fatalf("unknown field %q", tt.field) + } b := &bundle.Bundle{ Config: config.Root{ @@ -68,7 +99,13 @@ func TestConfigureClusterPolicyDefinition(t *testing.T) { } require.NoError(t, diags.Error()) - assert.Equal(t, tt.wantDefinition, b.Config.Resources.ClusterPolicies["pol"].Definition) + got := b.Config.Resources.ClusterPolicies["pol"] + switch tt.field { + case "policy_family_definition_overrides": + assert.Equal(t, tt.want, got.PolicyFamilyDefinitionOverrides) + case "definition": + assert.Equal(t, tt.want, got.Definition) + } }) } } diff --git a/bundle/config/resources/cluster_policy.go b/bundle/config/resources/cluster_policy.go index 98263d1a676..5c447b0beb9 100644 --- a/bundle/config/resources/cluster_policy.go +++ b/bundle/config/resources/cluster_policy.go @@ -19,6 +19,10 @@ type ClusterPolicy struct { // definition be authored as inline YAML; ConfigureClusterPolicyDefinition normalizes // it to a JSON string before deploy. Definition any `json:"definition,omitempty"` + + // Shadows the embedded compute.CreatePolicy.PolicyFamilyDefinitionOverrides (a string), + // same as Definition: also a policy document authorable as inline YAML. + PolicyFamilyDefinitionOverrides any `json:"policy_family_definition_overrides,omitempty"` } func (s *ClusterPolicy) UnmarshalJSON(b []byte) error { diff --git a/bundle/direct/dresources/cluster_policy.go b/bundle/direct/dresources/cluster_policy.go index e422a227616..ae69390b9cc 100644 --- a/bundle/direct/dresources/cluster_policy.go +++ b/bundle/direct/dresources/cluster_policy.go @@ -19,11 +19,13 @@ func (*ResourceClusterPolicy) New(client *databricks.WorkspaceClient) *ResourceC func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *compute.CreatePolicy { cp := input.CreatePolicy - // The top-level Definition shadows the embedded string; ConfigureClusterPolicyDefinition - // has already normalized it to a JSON string by this point. + // Copy the shadow fields, already normalized to JSON strings by ConfigureClusterPolicyDefinition. if s, ok := input.Definition.(string); ok { cp.Definition = s } + if s, ok := input.PolicyFamilyDefinitionOverrides.(string); ok { + cp.PolicyFamilyDefinitionOverrides = s + } return &cp } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 85de80de072..311426837f3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -619,7 +619,7 @@ }, "policy_family_definition_overrides": { "description": "Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).\nThe JSON document must be passed as a string and cannot be embedded in the requests.\n\nYou can use this to customize the policy definition inherited from the policy family.\nPolicy rules specified here are merged into the inherited policy definition.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/interface" }, "policy_family_id": { "description": "ID of the policy family. The cluster policy's policy definition inherits the policy\nfamily's policy definition.\n\nCannot be used with `definition`. Use `policy_family_definition_overrides` instead to\ncustomize the policy definition.", From 4330ddea30f4ad2c6fb6838c8258ced78d6fee14 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 12:38:26 +0000 Subject: [PATCH 34/44] regenerate file --- acceptance/bundle/refschema/out.fields.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index ac9a5e87d5e..f04d1416677 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -308,6 +308,7 @@ resources.cluster_policies.*.lifecycle.prevent_destroy bool INPUT resources.cluster_policies.*.max_clusters_per_user int64 ALL resources.cluster_policies.*.modified_status string INPUT resources.cluster_policies.*.name string ALL +resources.cluster_policies.*.policy_family_definition_overrides any INPUT resources.cluster_policies.*.policy_family_definition_overrides string ALL resources.cluster_policies.*.policy_family_id string ALL resources.cluster_policies.*.policy_id string REMOTE From 71f4f5ef09c469991179db8bbc680937f82bcf6e Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 12:44:16 +0000 Subject: [PATCH 35/44] delete cross bundle tests --- .../cross_bundle_id/bundle_a/databricks.yml | 8 -- .../cross_bundle_id/bundle_b/databricks.yml | 14 ---- .../cross_bundle_id/out.test.toml | 2 - .../cross_bundle_id/output.txt | 73 ------------------ .../cluster_policies/cross_bundle_id/script | 17 ----- .../cross_bundle_id/test.toml | 1 - .../bundle_a/databricks.yml | 14 ---- .../bundle_b/databricks.yml | 19 ----- .../cross_bundle_lookup/out.test.toml | 2 - .../cross_bundle_lookup/output.txt | 74 ------------------- .../cross_bundle_lookup/script | 17 ----- .../cross_bundle_lookup/test.toml | 1 - 12 files changed, 242 deletions(-) delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/script delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script delete mode 100644 acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml deleted file mode 100644 index 195681f9c74..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml +++ /dev/null @@ -1,8 +0,0 @@ -bundle: - name: cluster_policy_producer - -resources: - cluster_policies: - pol: - name: shared_policy - definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml deleted file mode 100644 index afacb0e118e..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml +++ /dev/null @@ -1,14 +0,0 @@ -bundle: - name: cluster_policy_consumer - -resources: - jobs: - j: - name: consumer_job - tasks: - - task_key: main - new_cluster: - # Replaced at test time with the policy id created by bundle_a. - policy_id: PLACEHOLDER_POLICY_ID - spark_version: 13.3.x-scala2.12 - num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml deleted file mode 100644 index 0938e678987..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt deleted file mode 100644 index 84da21ee84c..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt +++ /dev/null @@ -1,73 +0,0 @@ - -=== Bundle A creates the cluster policy ->>> withdir bundle_a [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Inject A's policy id into bundle B, then deploy B ->>> update_file.py bundle_b/databricks.yml PLACEHOLDER_POLICY_ID [POL_ID] - ->>> withdir bundle_b [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Policy created by A, then B's job carries that same policy id ->>> print_requests.py //policies/clusters //jobs -{ - "method": "POST", - "path": "/api/2.0/policies/clusters/create", - "body": { - "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "shared_policy" - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "consumer_job", - "queue": { - "enabled": true - }, - "tasks": [ - { - "new_cluster": { - "num_workers": 1, - "policy_id": "[POL_ID]", - "spark_version": "13.3.x-scala2.12" - }, - "task_key": "main" - } - ] - } -} - -=== Cleanup ->>> withdir bundle_b [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.j - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default - -Deleting files... -Destroy complete! - ->>> withdir bundle_a [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.cluster_policies.pol - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script deleted file mode 100644 index e2b47428da7..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script +++ /dev/null @@ -1,17 +0,0 @@ -title "Bundle A creates the cluster policy" -trace withdir bundle_a $CLI bundle deploy - -# Capture A's server-generated policy id and register [POL_ID]. -pol_id=$(withdir bundle_a read_id.py pol) - -title "Inject A's policy id into bundle B, then deploy B" -trace update_file.py bundle_b/databricks.yml PLACEHOLDER_POLICY_ID "$pol_id" -trace withdir bundle_b $CLI bundle deploy - -title "Policy created by A, then B's job carries that same policy id" -trace print_requests.py //policies/clusters //jobs - -title "Cleanup" -trace withdir bundle_b $CLI bundle destroy --auto-approve -trace withdir bundle_a $CLI bundle destroy --auto-approve -rm -f out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml deleted file mode 100644 index 601384fdf96..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml +++ /dev/null @@ -1 +0,0 @@ -Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml deleted file mode 100644 index 86501c512c3..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml +++ /dev/null @@ -1,14 +0,0 @@ -bundle: - name: cluster_policy_producer - -resources: - cluster_policies: - pol: - name: shared_lookup_policy - definition: |- - { - "spark_version": { - "type": "fixed", - "value": "13.3.x-scala2.12" - } - } diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml deleted file mode 100644 index 0789f610f03..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml +++ /dev/null @@ -1,19 +0,0 @@ -bundle: - name: cluster_policy_consumer - -variables: - policy: - description: Resolve the policy created by bundle_a by name. - lookup: - cluster_policy: shared_lookup_policy - -resources: - jobs: - j: - name: consumer_job - tasks: - - task_key: main - new_cluster: - policy_id: ${var.policy} - spark_version: 13.3.x-scala2.12 - num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml deleted file mode 100644 index 0938e678987..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt deleted file mode 100644 index b5061bbc3f3..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt +++ /dev/null @@ -1,74 +0,0 @@ - -=== Bundle A creates the cluster policy ->>> withdir bundle_a [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Bundle B resolves the policy by name via lookup ->>> withdir bundle_b [CLI] bundle validate -o json -{ - "policy": { - "description": "Resolve the policy created by bundle_a by name.", - "lookup": { - "cluster_policy": "shared_lookup_policy" - }, - "value": "[POL_ID]" - } -} - -=== Deploy B; its job carries the resolved policy id ->>> withdir bundle_b [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> print_requests.py //jobs -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "consumer_job", - "queue": { - "enabled": true - }, - "tasks": [ - { - "new_cluster": { - "num_workers": 1, - "policy_id": "[POL_ID]", - "spark_version": "13.3.x-scala2.12" - }, - "task_key": "main" - } - ] - } -} - -=== Cleanup ->>> withdir bundle_b [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.j - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default - -Deleting files... -Destroy complete! - ->>> withdir bundle_a [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.cluster_policies.pol - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script deleted file mode 100644 index affac0bb5bb..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script +++ /dev/null @@ -1,17 +0,0 @@ -title "Bundle A creates the cluster policy" -trace withdir bundle_a $CLI bundle deploy - -# Register [POL_ID] for A's server-generated policy id. -pol_id=$(withdir bundle_a read_id.py pol) - -title "Bundle B resolves the policy by name via lookup" -trace withdir bundle_b $CLI bundle validate -o json | jq '.variables' - -title "Deploy B; its job carries the resolved policy id" -trace withdir bundle_b $CLI bundle deploy -trace print_requests.py //jobs - -title "Cleanup" -trace withdir bundle_b $CLI bundle destroy --auto-approve -trace withdir bundle_a $CLI bundle destroy --auto-approve -rm -f out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml deleted file mode 100644 index 601384fdf96..00000000000 --- a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml +++ /dev/null @@ -1 +0,0 @@ -Ignore = [".databricks"] From dc91553e2edf95505b34c3c65215c3f2298aae3e Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 13:09:19 +0000 Subject: [PATCH 36/44] update goldens for direct deployments --- .../resources/cluster_policies/basic/output.txt | 15 +++++++-------- .../definition_multiline/output.txt | 9 ++++----- .../cluster_policies/definition_yaml/output.txt | 9 ++++----- .../cluster_policies/job_ref/output.txt | 17 +++++++++-------- .../out_of_band_change/output.txt | 15 +++++++-------- .../policy_family_overrides/output.txt | 11 ++++++----- 6 files changed, 37 insertions(+), 39 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt index 3d0358684e4..735ccb05b6d 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/output.txt +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -30,9 +30,9 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Created cluster_policies.test_cluster_policy +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request >>> jq select(.method == "POST" and (.path | contains("/policies/clusters/create"))) out.requests.txt @@ -62,9 +62,9 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Updated cluster_policies.test_cluster_policy +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the update request >>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt @@ -97,8 +97,7 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default -Deleting files... -Destroy complete! +Destroy: 1 deleted === Verify the destroy request >>> jq select(.method == "POST" and (.path | contains("/policies/clusters/delete"))) out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt index e6912527df8..f43b38553a9 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt @@ -9,9 +9,9 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Created cluster_policies.pol +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Create body preserves the block-scalar definition as a newline-escaped string >>> print_requests.py //policies/clusters @@ -30,5 +30,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default -Deleting files... -Destroy complete! +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt index edc38996d16..aefc3e991fa 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -9,9 +9,9 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Created cluster_policies.pol +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings) >>> print_requests.py //policies/clusters @@ -30,5 +30,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default -Deleting files... -Destroy complete! +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt index 00620492ca3..2a80317d90e 100644 --- a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -1,9 +1,10 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Created cluster_policies.pol +Created jobs.j +Files: 4 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id >>> print_requests.py //policies/clusters //jobs @@ -50,9 +51,10 @@ Deployment complete! >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Updated cluster_policies.pol +Updated jobs.j +Files: 3 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged >>> print_requests.py //policies/clusters //jobs { @@ -103,8 +105,7 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default -Deleting files... -Destroy complete! +Destroy: 2 deleted >>> print_requests.py //policies/clusters //jobs { diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt index 26c36bac429..897453ca753 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt @@ -1,9 +1,9 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Created cluster_policies.test_cluster_policy +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op immediately after deploy >>> [CLI] bundle plan @@ -21,9 +21,9 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Redeploy reconciles the policy back to the configured definition >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Updated cluster_policies.test_cluster_policy +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the edit request restored the configured definition >>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt @@ -47,5 +47,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default -Deleting files... -Destroy complete! +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt index fbc09a64dc5..3f87e0b314b 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt @@ -20,9 +20,11 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Created cluster_policies.json_string +Created cluster_policies.multiline +Created cluster_policies.yaml +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved) >>> print_requests.py //policies/clusters/create --sort @@ -62,5 +64,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default -Deleting files... -Destroy complete! +Destroy: 3 deleted From 082450e687d3d3dee2e75f22e6ab6b542a916c44 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 18 Aug 2026 09:24:30 +0000 Subject: [PATCH 37/44] check for backend normalization --- .../backend_normalization/databricks.yml | 31 +++++++++++++++ .../backend_normalization/out.test.toml | 2 + .../backend_normalization/output.txt | 39 +++++++++++++++++++ .../backend_normalization/script | 19 +++++++++ .../backend_normalization/test.toml | 7 ++++ 5 files changed, 98 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/backend_normalization/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/backend_normalization/script create mode 100644 acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml b/acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml new file mode 100644 index 00000000000..33b9cb0edd9 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml @@ -0,0 +1,31 @@ +bundle: + name: cluster_policy_backend_normalization + +resources: + cluster_policies: + # Definition as a compact JSON string. + json_string: + name: probe_json_string + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + + # Definition as a multiline block-scalar JSON string. + multiline: + name: probe_multiline + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } + + # Definition as native YAML (normalized to compact JSON by the mutator before deploy). + yaml: + name: probe_yaml + definition: + spark_version: + type: fixed + value: 13.3.x-scala2.12 + autotermination_minutes: + type: range + maxValue: 120 diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/out.test.toml b/acceptance/bundle/resources/cluster_policies/backend_normalization/out.test.toml new file mode 100644 index 00000000000..c502b28221b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt b/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt new file mode 100644 index 00000000000..8304f30750e --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt @@ -0,0 +1,39 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_backend_normalization/default/files... +Created cluster_policies.json_string +Created cluster_policies.multiline +Created cluster_policies.yaml +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== [json_string] definition returned by the backend (GET) +>>> [CLI] cluster-policies get [JSON_STRING_ID] -o json +{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}} + +=== [multiline] definition returned by the backend (GET) +>>> [CLI] cluster-policies get [MULTILINE_ID] -o json +{ + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } +} + +=== [yaml] definition returned by the backend (GET) +>>> [CLI] cluster-policies get [YAML_ID] -o json +{"autotermination_minutes":{"maxValue":120,"type":"range"},"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}} + +=== Plan after deploy: any drift means the backend reformatted a definition +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.json_string + delete resources.cluster_policies.multiline + delete resources.cluster_policies.yaml + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_backend_normalization/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/script b/acceptance/bundle/resources/cluster_policies/backend_normalization/script new file mode 100644 index 00000000000..d0b763e2ad4 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/script @@ -0,0 +1,19 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +# For each authoring style, print the definition string the backend returns on GET. +# Compare it against what we sent to see whether the backend reformats the JSON. +for name in json_string multiline yaml; do + pol_id=$(read_id.py "$name") + title "[$name] definition returned by the backend (GET)" + trace $CLI cluster-policies get "$pol_id" -o json | jq -r '.definition' +done + +# yaml is normalized to compact JSON by the mutator, matching the string the backend +# stores and returns, so it does not drift (plan below is a no-op). +title "Plan after deploy: any drift means the backend reformatted a definition" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml b/acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml new file mode 100644 index 00000000000..1e0b1e5cf6f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +Ignore = [ + ".databricks", + "databricks.yml", +] From 4a20bf56d8d632cc45d0b3e958c7beeea67f4516 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 18 Aug 2026 09:36:39 +0000 Subject: [PATCH 38/44] update tests to run on cloud --- .../resources/cluster_policies/basic/out.test.toml | 2 +- .../bundle/resources/cluster_policies/basic/output.txt | 10 +++++----- .../bundle/resources/cluster_policies/basic/script | 4 ++++ .../bundle/resources/cluster_policies/basic/test.toml | 1 + .../cluster_policies/definition_yaml/out.test.toml | 2 +- .../cluster_policies/definition_yaml/output.txt | 2 +- .../cluster_policies/definition_yaml/test.toml | 1 + 7 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/resources/cluster_policies/basic/test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/basic/out.test.toml b/acceptance/bundle/resources/cluster_policies/basic/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/basic/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt index 735ccb05b6d..877ec88d384 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/output.txt +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -31,7 +31,7 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... Created cluster_policies.test_cluster_policy -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request @@ -55,7 +55,7 @@ Resources: Cluster Policies: test_cluster_policy: Name: my_cluster_policy - URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID] === Update the cluster policy name >>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 @@ -74,7 +74,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "body": { "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", "name": "my_cluster_policy_2", - "policy_id": "[UUID]" + "policy_id": "[TEST_CLUSTER_POLICY_ID]" } } @@ -88,7 +88,7 @@ Resources: Cluster Policies: test_cluster_policy: Name: my_cluster_policy_2 - URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID] === Destroy the cluster policy >>> [CLI] bundle destroy --auto-approve @@ -105,7 +105,7 @@ Destroy: 1 deleted "method": "POST", "path": "/api/2.0/policies/clusters/delete", "body": { - "policy_id": "[UUID]" + "policy_id": "[TEST_CLUSTER_POLICY_ID]" } } diff --git a/acceptance/bundle/resources/cluster_policies/basic/script b/acceptance/bundle/resources/cluster_policies/basic/script index 412bdcf1520..342e3b69183 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/script +++ b/acceptance/bundle/resources/cluster_policies/basic/script @@ -10,6 +10,10 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy +# Mask the server-assigned policy id: the real backend returns a hex id that the +# built-in UUID replacement misses, so register it explicitly for cloud parity. +read_id.py test_cluster_policy > /dev/null + title "Verify the create request" trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/create")))' out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/basic/test.toml b/acceptance/bundle/resources/cluster_policies/basic/test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt index aefc3e991fa..fe8cd5f90c6 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -10,7 +10,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files... Created cluster_policies.pol -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings) diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml b/acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml @@ -0,0 +1 @@ +Cloud = true From 4f467bf59785ebdd97a52e7c6c019eddcfd5db10 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 18 Aug 2026 10:41:01 +0000 Subject: [PATCH 39/44] use print_requests.py instead of inline jq in cluster_policy tests Co-authored-by: Isaac --- .../bundle/resources/cluster_policies/basic/output.txt | 6 +++--- acceptance/bundle/resources/cluster_policies/basic/script | 8 ++++---- .../cluster_policies/out_of_band_change/output.txt | 2 +- .../resources/cluster_policies/out_of_band_change/script | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt index 877ec88d384..52f258f3cd5 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/output.txt +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -35,7 +35,7 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request ->>> jq select(.method == "POST" and (.path | contains("/policies/clusters/create"))) out.requests.txt +>>> print_requests.py //policies/clusters/create { "method": "POST", "path": "/api/2.0/policies/clusters/create", @@ -67,7 +67,7 @@ Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the update request ->>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt +>>> print_requests.py //policies/clusters/edit { "method": "POST", "path": "/api/2.0/policies/clusters/edit", @@ -100,7 +100,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Verify the destroy request ->>> jq select(.method == "POST" and (.path | contains("/policies/clusters/delete"))) out.requests.txt +>>> print_requests.py //policies/clusters/delete { "method": "POST", "path": "/api/2.0/policies/clusters/delete", diff --git a/acceptance/bundle/resources/cluster_policies/basic/script b/acceptance/bundle/resources/cluster_policies/basic/script index 342e3b69183..8b14f6ef39b 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/script +++ b/acceptance/bundle/resources/cluster_policies/basic/script @@ -5,7 +5,7 @@ trace $CLI bundle summary cleanup() { trace $CLI bundle destroy --auto-approve - rm out.requests.txt + rm -f out.requests.txt } trap cleanup EXIT trace $CLI bundle deploy @@ -15,7 +15,7 @@ trace $CLI bundle deploy read_id.py test_cluster_policy > /dev/null title "Verify the create request" -trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/create")))' out.requests.txt +trace print_requests.py //policies/clusters/create trace $CLI bundle summary @@ -24,7 +24,7 @@ trace update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 trace $CLI bundle deploy title "Verify the update request" -trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt +trace print_requests.py //policies/clusters/edit trace $CLI bundle summary @@ -32,6 +32,6 @@ title "Destroy the cluster policy" trace $CLI bundle destroy --auto-approve title "Verify the destroy request" -trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/delete")))' out.requests.txt +trace print_requests.py //policies/clusters/delete trace $CLI bundle summary diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt index 897453ca753..61e3763caa0 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt @@ -26,7 +26,7 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the edit request restored the configured definition ->>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt +>>> print_requests.py //policies/clusters/edit { "method": "POST", "path": "/api/2.0/policies/clusters/edit", diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script index 47d8585c72d..eeecf83698c 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script @@ -28,7 +28,7 @@ title "Redeploy reconciles the policy back to the configured definition" trace $CLI bundle deploy title "Verify the edit request restored the configured definition" -trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt +trace print_requests.py //policies/clusters/edit title "Plan is a no-op again" trace $CLI bundle plan From f9e55267e0244d7bb94bdd64e3b6d587a2081527 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 19 Aug 2026 17:53:33 +0000 Subject: [PATCH 40/44] add backend_defaults because of policy_family_definition_overrides --- .../policy_family_definition/databricks.yml | 10 +++++++ .../policy_family_definition/out.test.toml | 2 ++ .../policy_family_definition/output.txt | 26 +++++++++++++++++++ .../policy_family_definition/script | 19 ++++++++++++++ .../policy_family_definition/test.toml | 4 +++ bundle/direct/dresources/resources.yml | 8 ++++++ libs/testserver/cluster_policies.go | 16 ++++++++++++ 7 files changed, 85 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_definition/script create mode 100644 acceptance/bundle/resources/cluster_policies/policy_family_definition/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml b/acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml new file mode 100644 index 00000000000..b5a81a98c29 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: cluster_policy_family + +resources: + cluster_policies: + # Authored from a policy family with no definition. The backend computes the + # definition from the family and returns it on read; the config leaves it empty. + family_policy: + name: my_family_policy + policy_family_id: personal-vm diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt new file mode 100644 index 00000000000..fbe6ff7d962 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt @@ -0,0 +1,26 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_family/default/files... +Created cluster_policies.family_policy +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Plan is a no-op after deploy (server-computed definition is not drift) +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Redeploy issues no edit +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_family/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //policies/clusters/edit + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.family_policy + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_family/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/script b/acceptance/bundle/resources/cluster_policies/policy_family_definition/script new file mode 100644 index 00000000000..041da8c22fc --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/script @@ -0,0 +1,19 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle deploy + +# The backend computes `definition` from the policy family and returns it on read, +# even though the config leaves definition empty. Because definition is declared as +# a backend default, the server-computed value must not register as drift: without +# it every plan would show an update and every deploy would re-issue an edit. +title "Plan is a no-op after deploy (server-computed definition is not drift)" +trace $CLI bundle plan + +title "Redeploy issues no edit" +rm -f out.requests.txt +trace $CLI bundle deploy +trace print_requests.py //policies/clusters/edit diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/test.toml b/acceptance/bundle/resources/cluster_policies/policy_family_definition/test.toml new file mode 100644 index 00000000000..b03f24bc233 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/test.toml @@ -0,0 +1,4 @@ +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index e98a1ac2330..3f764a8ac4f 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -647,6 +647,14 @@ resources: # DataSecurityModeDiffSuppressFunc: suppress when old != "" && new == "" #- field: data_security_mode + cluster_policies: + backend_defaults: + # A policy authored with policy_family_id and no definition gets its definition + # computed from the policy family by the backend and returned on read. Config + # leaves definition empty, so old/new are nil and backend_defaults skips the + # server-computed remote value; without this every deploy sees drift and re-Edits. + - field: definition + instance_pools: # Field behaviors follow the TF provider tags cross-referenced with the edit API (compute.EditInstancePool): # https://github.com/databricks/terraform-provider-databricks/blob/main/pools/resource_instance_pool.go diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index 5616a32a00f..12b5995f09c 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -8,6 +8,15 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" ) +// policyFamilyDefinition mimics the real backend: a policy created from a policy +// family has its definition computed from the family and returned on read, even +// though the config never sets definition. One fixed key is enough to reproduce a +// non-empty server-computed definition so tests exercise the backend_defaults +// suppression for definition (see resources.yml cluster_policies). +func policyFamilyDefinition(familyID string) string { + return fmt.Sprintf(`{"policy_family":{"type":"fixed","value":%q}}`, familyID) +} + func (s *FakeWorkspace) ClusterPoliciesCreate(req Request) any { // Unmarshal into the stored (GET) type directly: CreatePolicy and Policy // share JSON field names, so every config field is carried over. @@ -16,6 +25,10 @@ func (s *FakeWorkspace) ClusterPoliciesCreate(req Request) any { return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} } + if policy.Definition == "" && policy.PolicyFamilyId != "" { + policy.Definition = policyFamilyDefinition(policy.PolicyFamilyId) + } + defer s.LockUnlock()() id := nextUUID() @@ -75,6 +88,9 @@ func (s *FakeWorkspace) ClusterPoliciesEdit(req Request) any { policy.MaxClustersPerUser = request.MaxClustersPerUser policy.PolicyFamilyDefinitionOverrides = request.PolicyFamilyDefinitionOverrides policy.PolicyFamilyId = request.PolicyFamilyId + if policy.Definition == "" && policy.PolicyFamilyId != "" { + policy.Definition = policyFamilyDefinition(policy.PolicyFamilyId) + } s.ClusterPolicies[request.PolicyId] = policy return Response{} From f9685697083c373f014bdfee91c27cbcc3b74c9d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 24 Aug 2026 13:54:41 +0000 Subject: [PATCH 41/44] update tests to use default cloud=true --- .../cluster_policies/backend_normalization/test.toml | 1 - .../bundle/resources/cluster_policies/basic/output.txt | 2 +- .../bundle/resources/cluster_policies/basic/test.toml | 1 - .../definition_multiline/out.test.toml | 2 +- .../cluster_policies/definition_yaml/output.txt | 2 +- .../cluster_policies/definition_yaml/test.toml | 1 - .../resources/cluster_policies/job_ref/output.txt | 2 +- .../resources/cluster_policies/job_ref/test.toml | 4 ++++ .../cluster_policies/out_of_band_change/databricks.yml | 5 +++-- .../cluster_policies/out_of_band_change/out.test.toml | 2 +- .../cluster_policies/out_of_band_change/output.txt | 10 +++++----- .../cluster_policies/out_of_band_change/script | 2 +- .../policy_family_definition/out.test.toml | 2 +- .../policy_family_overrides/out.test.toml | 2 +- acceptance/bundle/resources/cluster_policies/test.toml | 2 +- 15 files changed, 21 insertions(+), 19 deletions(-) delete mode 100644 acceptance/bundle/resources/cluster_policies/basic/test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/job_ref/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml b/acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml index 1e0b1e5cf6f..435ad6e2630 100644 --- a/acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml @@ -1,4 +1,3 @@ -Cloud = true RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [ diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt index 52f258f3cd5..0a87f97c7ef 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/output.txt +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -31,7 +31,7 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... Created cluster_policies.test_cluster_policy -Files: 5 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request diff --git a/acceptance/bundle/resources/cluster_policies/basic/test.toml b/acceptance/bundle/resources/cluster_policies/basic/test.toml deleted file mode 100644 index c7c6f58ed6e..00000000000 --- a/acceptance/bundle/resources/cluster_policies/basic/test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = true diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt index fe8cd5f90c6..aefc3e991fa 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -10,7 +10,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files... Created cluster_policies.pol -Files: 5 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings) diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml b/acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml deleted file mode 100644 index c7c6f58ed6e..00000000000 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = true diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt index 2a80317d90e..1a5b87c1627 100644 --- a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -3,7 +3,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... Created cluster_policies.pol Created jobs.j -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/test.toml b/acceptance/bundle/resources/cluster_policies/job_ref/test.toml new file mode 100644 index 00000000000..7c7ba8ececc --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/test.toml @@ -0,0 +1,4 @@ +# The job task defines only new_cluster and no task type (notebook_task, etc.), so the +# real Jobs API rejects it ("No task defined"); the testserver doesn't validate. The task +# type is irrelevant here since the test only checks policy-id resolution and create ordering. +Cloud = false diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml index 5156a2b9b0f..b830ac67d42 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml @@ -1,8 +1,9 @@ bundle: - name: test_cluster_policy + # Unique bundle + policy name so this coexists with basic/ on a shared cloud workspace. + name: test_cluster_policy_oob resources: cluster_policies: test_cluster_policy: - name: my_cluster_policy + name: my_cluster_policy_oob definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt index 61e3763caa0..0ab16d8e8c0 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy_oob/default/files... Created cluster_policies.test_cluster_policy Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged @@ -10,7 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Edit the policy definition out of band ->>> [CLI] cluster-policies edit --json {"policy_id":"[TEST_CLUSTER_POLICY_ID]","name":"my_cluster_policy","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"} +>>> [CLI] cluster-policies edit --json {"policy_id":"[TEST_CLUSTER_POLICY_ID]","name":"my_cluster_policy_oob","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"} === Plan detects the drift >>> [CLI] bundle plan @@ -20,7 +20,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Redeploy reconciles the policy back to the configured definition >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy_oob/default/files... Updated cluster_policies.test_cluster_policy Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged @@ -32,7 +32,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "path": "/api/2.0/policies/clusters/edit", "body": { "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "my_cluster_policy", + "name": "my_cluster_policy_oob", "policy_id": "[TEST_CLUSTER_POLICY_ID]" } } @@ -45,6 +45,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged The following resources will be deleted: delete resources.cluster_policies.test_cluster_policy -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy_oob/default Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script index eeecf83698c..2b1b74175df 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script @@ -15,7 +15,7 @@ policy_id="$(read_id.py test_cluster_policy)" # API, the way an admin would in the UI, without touching databricks.yml. The # recorded bundle state is now stale, so the next plan must detect the drift. title "Edit the policy definition out of band" -trace $CLI cluster-policies edit --json '{"policy_id":"'"$policy_id"'","name":"my_cluster_policy","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"}' +trace $CLI cluster-policies edit --json '{"policy_id":"'"$policy_id"'","name":"my_cluster_policy_oob","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"}' # Discard the out-of-band request so the verification below captures only the # reconciling edit issued by the redeploy. diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/test.toml b/acceptance/bundle/resources/cluster_policies/test.toml index 3fe510e7c1b..d9b1de47061 100644 --- a/acceptance/bundle/resources/cluster_policies/test.toml +++ b/acceptance/bundle/resources/cluster_policies/test.toml @@ -1,4 +1,4 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [ From 53ccee4c5833ab9b8f151bf8309c812ba2230c88 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 24 Aug 2026 14:06:49 +0000 Subject: [PATCH 42/44] add test for define both --- .../definition_and_family_conflict/databricks.yml | 10 ++++++++++ .../definition_and_family_conflict/out.test.toml | 2 ++ .../definition_and_family_conflict/output.txt | 11 +++++++++++ .../definition_and_family_conflict/script | 7 +++++++ .../definition_and_family_conflict/test.toml | 2 ++ libs/testserver/cluster_policies.go | 11 +++++++++++ 6 files changed, 43 insertions(+) create mode 100644 acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml create mode 100644 acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script create mode 100644 acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml new file mode 100644 index 00000000000..1337e34cfe9 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: cluster_policy_definition_and_family_conflict + +resources: + cluster_policies: + pol: + name: my_conflict_policy + # definition and policy_family_id are mutually exclusive; the backend rejects both. + policy_family_id: personal-vm + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/out.test.toml new file mode 100644 index 00000000000..c502b28221b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt new file mode 100644 index 00000000000..1af0c6ced45 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt @@ -0,0 +1,11 @@ + +=== Deploying a policy with both definition and policy_family_id must fail +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_and_family_conflict/default/files... +Error: cannot create resources.cluster_policies.pol: policy_family_id and definition cannot be used together (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/policies/clusters/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: policy_family_id and definition cannot be used together + +Files: 4 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script new file mode 100644 index 00000000000..55127b542b1 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script @@ -0,0 +1,7 @@ +cleanup() { + $CLI bundle destroy --auto-approve &> LOG.destroy || true +} +trap cleanup EXIT + +title "Deploying a policy with both definition and policy_family_id must fail\n" +musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/test.toml b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/test.toml new file mode 100644 index 00000000000..7b7a2740b28 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/test.toml @@ -0,0 +1,2 @@ +# This test only asserts the create error; recorded requests are noisy and differ by env. +RecordRequests = false diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index 12b5995f09c..13c48f0770e 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -25,6 +25,17 @@ func (s *FakeWorkspace) ClusterPoliciesCreate(req Request) any { return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} } + // The backend rejects definition and policy_family_id together. + if policy.Definition != "" && policy.PolicyFamilyId != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "policy_family_id and definition cannot be used together", + }, + } + } + if policy.Definition == "" && policy.PolicyFamilyId != "" { policy.Definition = policyFamilyDefinition(policy.PolicyFamilyId) } From c1da77961f162c921aad828f6b3a618814b6c4b3 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 24 Aug 2026 15:07:22 +0000 Subject: [PATCH 43/44] use UNIQUE_NAME in cluster_policy cloud tests to avoid name collisions Cluster policy names are workspace-global and the CI workspace is shared across concurrent runs, so fixed policy names collided ("already exists"). Author names and the bundle root_path with $UNIQUE_NAME via databricks.yml.tmpl + envsubst so each run is isolated. Co-authored-by: Isaac --- .../{databricks.yml => databricks.yml.tmpl} | 9 ++++-- .../backend_normalization/output.txt | 6 ++-- .../backend_normalization/script | 4 ++- .../{databricks.yml => databricks.yml.tmpl} | 5 ++- .../cluster_policies/basic/output.txt | 32 +++++++++---------- .../resources/cluster_policies/basic/script | 4 ++- .../{databricks.yml => databricks.yml.tmpl} | 5 ++- .../definition_and_family_conflict/output.txt | 4 +-- .../definition_and_family_conflict/script | 4 ++- .../{databricks.yml => databricks.yml.tmpl} | 5 ++- .../definition_multiline/output.txt | 10 +++--- .../definition_multiline/script | 4 ++- .../{databricks.yml => databricks.yml.tmpl} | 5 ++- .../definition_yaml/output.txt | 10 +++--- .../cluster_policies/definition_yaml/script | 4 ++- .../{databricks.yml => databricks.yml.tmpl} | 6 ++-- .../out_of_band_change/output.txt | 12 +++---- .../out_of_band_change/script | 6 ++-- .../{databricks.yml => databricks.yml.tmpl} | 5 ++- .../policy_family_definition/output.txt | 8 ++--- .../policy_family_definition/script | 4 ++- .../{databricks.yml => databricks.yml.tmpl} | 9 ++++-- .../policy_family_overrides/output.txt | 18 +++++------ .../policy_family_overrides/script | 4 ++- 24 files changed, 111 insertions(+), 72 deletions(-) rename acceptance/bundle/resources/cluster_policies/backend_normalization/{databricks.yml => databricks.yml.tmpl} (81%) rename acceptance/bundle/resources/cluster_policies/basic/{databricks.yml => databricks.yml.tmpl} (65%) rename acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/{databricks.yml => databricks.yml.tmpl} (77%) rename acceptance/bundle/resources/cluster_policies/definition_multiline/{databricks.yml => databricks.yml.tmpl} (74%) rename acceptance/bundle/resources/cluster_policies/definition_yaml/{databricks.yml => databricks.yml.tmpl} (81%) rename acceptance/bundle/resources/cluster_policies/out_of_band_change/{databricks.yml => databricks.yml.tmpl} (59%) rename acceptance/bundle/resources/cluster_policies/policy_family_definition/{databricks.yml => databricks.yml.tmpl} (76%) rename acceptance/bundle/resources/cluster_policies/policy_family_overrides/{databricks.yml => databricks.yml.tmpl} (84%) diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml b/acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml.tmpl similarity index 81% rename from acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml rename to acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml.tmpl index 33b9cb0edd9..8e0c536a816 100644 --- a/acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml.tmpl @@ -1,16 +1,19 @@ bundle: name: cluster_policy_backend_normalization +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: # Definition as a compact JSON string. json_string: - name: probe_json_string + name: probe_json_string-$UNIQUE_NAME definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' # Definition as a multiline block-scalar JSON string. multiline: - name: probe_multiline + name: probe_multiline-$UNIQUE_NAME definition: |- { "spark_version": { @@ -21,7 +24,7 @@ resources: # Definition as native YAML (normalized to compact JSON by the mutator before deploy). yaml: - name: probe_yaml + name: probe_yaml-$UNIQUE_NAME definition: spark_version: type: fixed diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt b/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt index 8304f30750e..994a3c0fc9f 100644 --- a/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt @@ -1,10 +1,10 @@ >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_backend_normalization/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.json_string Created cluster_policies.multiline Created cluster_policies.yaml -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === [json_string] definition returned by the backend (GET) @@ -34,6 +34,6 @@ The following resources will be deleted: delete resources.cluster_policies.multiline delete resources.cluster_policies.yaml -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_backend_normalization/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 3 deleted diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/script b/acceptance/bundle/resources/cluster_policies/backend_normalization/script index d0b763e2ad4..50f10926dd2 100644 --- a/acceptance/bundle/resources/cluster_policies/backend_normalization/script +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { trace $CLI bundle destroy --auto-approve } @@ -16,4 +18,4 @@ done # yaml is normalized to compact JSON by the mutator, matching the string the backend # stores and returns, so it does not drift (plan below is a no-op). title "Plan after deploy: any drift means the backend reformatted a definition" -trace $CLI bundle plan +trace $CLI bundle plan \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/basic/databricks.yml b/acceptance/bundle/resources/cluster_policies/basic/databricks.yml.tmpl similarity index 65% rename from acceptance/bundle/resources/cluster_policies/basic/databricks.yml rename to acceptance/bundle/resources/cluster_policies/basic/databricks.yml.tmpl index 5156a2b9b0f..3009f5a3b05 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/basic/databricks.yml.tmpl @@ -1,8 +1,11 @@ bundle: name: test_cluster_policy +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: test_cluster_policy: - name: my_cluster_policy + name: my_cluster_policy-$UNIQUE_NAME definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt index 0a87f97c7ef..9141e7bf3cc 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/output.txt +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -4,7 +4,7 @@ Name: test_cluster_policy Target: default Workspace: User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Validation OK! @@ -12,7 +12,7 @@ Validation OK! { "test_cluster_policy": { "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "my_cluster_policy" + "name": "my_cluster_policy-[UNIQUE_NAME]" } } @@ -21,17 +21,17 @@ Name: test_cluster_policy Target: default Workspace: User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Resources: Cluster Policies: test_cluster_policy: - Name: my_cluster_policy + Name: my_cluster_policy-[UNIQUE_NAME] URL: (not deployed) >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.test_cluster_policy -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request @@ -41,7 +41,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "path": "/api/2.0/policies/clusters/create", "body": { "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "my_cluster_policy" + "name": "my_cluster_policy-[UNIQUE_NAME]" } } @@ -50,18 +50,18 @@ Name: test_cluster_policy Target: default Workspace: User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Resources: Cluster Policies: test_cluster_policy: - Name: my_cluster_policy + Name: my_cluster_policy-[UNIQUE_NAME] URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID] === Update the cluster policy name >>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated cluster_policies.test_cluster_policy Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged @@ -73,7 +73,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "path": "/api/2.0/policies/clusters/edit", "body": { "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "my_cluster_policy_2", + "name": "my_cluster_policy_2-[UNIQUE_NAME]", "policy_id": "[TEST_CLUSTER_POLICY_ID]" } } @@ -83,11 +83,11 @@ Name: test_cluster_policy Target: default Workspace: User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Resources: Cluster Policies: test_cluster_policy: - Name: my_cluster_policy_2 + Name: my_cluster_policy_2-[UNIQUE_NAME] URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID] === Destroy the cluster policy @@ -95,7 +95,7 @@ Resources: The following resources will be deleted: delete resources.cluster_policies.test_cluster_policy -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 1 deleted @@ -114,11 +114,11 @@ Name: test_cluster_policy Target: default Workspace: User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Resources: Cluster Policies: test_cluster_policy: - Name: my_cluster_policy_2 + Name: my_cluster_policy_2-[UNIQUE_NAME] URL: (not deployed) >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/cluster_policies/basic/script b/acceptance/bundle/resources/cluster_policies/basic/script index 8b14f6ef39b..82935208f7e 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/script +++ b/acceptance/bundle/resources/cluster_policies/basic/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + trace $CLI bundle validate trace $CLI bundle validate -o json | jq ".resources.cluster_policies" @@ -34,4 +36,4 @@ trace $CLI bundle destroy --auto-approve title "Verify the destroy request" trace print_requests.py //policies/clusters/delete -trace $CLI bundle summary +trace $CLI bundle summary \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml.tmpl similarity index 77% rename from acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml rename to acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml.tmpl index 1337e34cfe9..99b05c89fa3 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/databricks.yml.tmpl @@ -1,10 +1,13 @@ bundle: name: cluster_policy_definition_and_family_conflict +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: pol: - name: my_conflict_policy + name: my_conflict_policy-$UNIQUE_NAME # definition and policy_family_id are mutually exclusive; the backend rejects both. policy_family_id: personal-vm definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt index 1af0c6ced45..22279ae6568 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/output.txt @@ -1,6 +1,6 @@ === Deploying a policy with both definition and policy_family_id must fail -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_and_family_conflict/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Error: cannot create resources.cluster_policies.pol: policy_family_id and definition cannot be used together (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.0/policies/clusters/create @@ -8,4 +8,4 @@ HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE API message: policy_family_id and definition cannot be used together -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script index 55127b542b1..601a9520af1 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script @@ -1,7 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { $CLI bundle destroy --auto-approve &> LOG.destroy || true } trap cleanup EXIT title "Deploying a policy with both definition and policy_family_id must fail\n" -musterr $CLI bundle deploy +musterr $CLI bundle deploy \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml.tmpl similarity index 74% rename from acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml rename to acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml.tmpl index 395e21aa5d0..b4d3637227c 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml.tmpl @@ -1,10 +1,13 @@ bundle: name: cluster_policy_definition_multiline +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: pol: - name: my_policy + name: my_policy-$UNIQUE_NAME definition: |- { "spark_version": { diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt index f43b38553a9..1436fd014e5 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt @@ -3,14 +3,14 @@ { "pol": { "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", - "name": "my_policy" + "name": "my_policy-[UNIQUE_NAME]" } } >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.pol -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Create body preserves the block-scalar definition as a newline-escaped string @@ -20,7 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "path": "/api/2.0/policies/clusters/create", "body": { "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", - "name": "my_policy" + "name": "my_policy-[UNIQUE_NAME]" } } @@ -28,6 +28,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged The following resources will be deleted: delete resources.cluster_policies.pol -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/script b/acceptance/bundle/resources/cluster_policies/definition_multiline/script index c134016b6e7..8d0571f508f 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_multiline/script +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + trace $CLI bundle validate -o json | jq ".resources.cluster_policies" trace $CLI bundle deploy @@ -6,4 +8,4 @@ title "Create body preserves the block-scalar definition as a newline-escaped st trace print_requests.py //policies/clusters trace $CLI bundle destroy --auto-approve -rm out.requests.txt +rm out.requests.txt \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml.tmpl similarity index 81% rename from acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml rename to acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml.tmpl index 49ecc474a22..8ee11375ead 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml.tmpl @@ -1,10 +1,13 @@ bundle: name: cluster_policy_definition_yaml +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: pol: - name: my_policy + name: my_policy-$UNIQUE_NAME definition: spark_version: type: fixed diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt index aefc3e991fa..c5edec39e9a 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -3,14 +3,14 @@ { "pol": { "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "my_policy" + "name": "my_policy-[UNIQUE_NAME]" } } >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.pol -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings) @@ -20,7 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "path": "/api/2.0/policies/clusters/create", "body": { "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "my_policy" + "name": "my_policy-[UNIQUE_NAME]" } } @@ -28,6 +28,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged The following resources will be deleted: delete resources.cluster_policies.pol -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/script b/acceptance/bundle/resources/cluster_policies/definition_yaml/script index 083c796fd5d..b087604b4d6 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/script +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + trace $CLI bundle validate -o json | jq ".resources.cluster_policies" trace $CLI bundle deploy @@ -6,4 +8,4 @@ title "Native YAML definition serializes to compact JSON, preserving numbers and trace print_requests.py //policies/clusters trace $CLI bundle destroy --auto-approve -rm out.requests.txt +rm out.requests.txt \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml.tmpl similarity index 59% rename from acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml rename to acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml.tmpl index b830ac67d42..ba5bd2d6d7d 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml.tmpl @@ -1,9 +1,11 @@ bundle: - # Unique bundle + policy name so this coexists with basic/ on a shared cloud workspace. name: test_cluster_policy_oob +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: test_cluster_policy: - name: my_cluster_policy_oob + name: my_cluster_policy_oob-$UNIQUE_NAME definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt index 0ab16d8e8c0..8f8afc280e4 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt @@ -1,8 +1,8 @@ >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy_oob/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.test_cluster_policy -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op immediately after deploy @@ -10,7 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Edit the policy definition out of band ->>> [CLI] cluster-policies edit --json {"policy_id":"[TEST_CLUSTER_POLICY_ID]","name":"my_cluster_policy_oob","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"} +>>> [CLI] cluster-policies edit --json {"policy_id":"[TEST_CLUSTER_POLICY_ID]","name":"my_cluster_policy_oob-[UNIQUE_NAME]","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"} === Plan detects the drift >>> [CLI] bundle plan @@ -20,7 +20,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Redeploy reconciles the policy back to the configured definition >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy_oob/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated cluster_policies.test_cluster_policy Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged @@ -32,7 +32,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "path": "/api/2.0/policies/clusters/edit", "body": { "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", - "name": "my_cluster_policy_oob", + "name": "my_cluster_policy_oob-[UNIQUE_NAME]", "policy_id": "[TEST_CLUSTER_POLICY_ID]" } } @@ -45,6 +45,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged The following resources will be deleted: delete resources.cluster_policies.test_cluster_policy -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy_oob/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script index 2b1b74175df..1d7f9ed72c3 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt @@ -15,7 +17,7 @@ policy_id="$(read_id.py test_cluster_policy)" # API, the way an admin would in the UI, without touching databricks.yml. The # recorded bundle state is now stale, so the next plan must detect the drift. title "Edit the policy definition out of band" -trace $CLI cluster-policies edit --json '{"policy_id":"'"$policy_id"'","name":"my_cluster_policy_oob","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"}' +trace $CLI cluster-policies edit --json '{"policy_id":"'"$policy_id"'","name":"my_cluster_policy_oob-'"$UNIQUE_NAME"'","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"}' # Discard the out-of-band request so the verification below captures only the # reconciling edit issued by the redeploy. @@ -31,4 +33,4 @@ title "Verify the edit request restored the configured definition" trace print_requests.py //policies/clusters/edit title "Plan is a no-op again" -trace $CLI bundle plan +trace $CLI bundle plan \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml b/acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml.tmpl similarity index 76% rename from acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml rename to acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml.tmpl index b5a81a98c29..d12ce7f8631 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/databricks.yml.tmpl @@ -1,10 +1,13 @@ bundle: name: cluster_policy_family +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: # Authored from a policy family with no definition. The backend computes the # definition from the family and returns it on read; the config leaves it empty. family_policy: - name: my_family_policy + name: my_family_policy-$UNIQUE_NAME policy_family_id: personal-vm diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt index fbe6ff7d962..7bad908a6af 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt @@ -1,8 +1,8 @@ >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_family/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.family_policy -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op after deploy (server-computed definition is not drift) @@ -11,7 +11,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Redeploy issues no edit >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_family/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged @@ -21,6 +21,6 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged The following resources will be deleted: delete resources.cluster_policies.family_policy -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_family/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/script b/acceptance/bundle/resources/cluster_policies/policy_family_definition/script index 041da8c22fc..29c09952964 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_definition/script +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt @@ -16,4 +18,4 @@ trace $CLI bundle plan title "Redeploy issues no edit" rm -f out.requests.txt trace $CLI bundle deploy -trace print_requests.py //policies/clusters/edit +trace print_requests.py //policies/clusters/edit \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml.tmpl similarity index 84% rename from acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml rename to acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml.tmpl index a50210d97a6..8dae1b8085c 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml.tmpl @@ -1,17 +1,20 @@ bundle: name: cluster_policy_overrides +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + resources: cluster_policies: # Overrides as a compact JSON string. json_string: - name: policy_json_string + name: policy_json_string-$UNIQUE_NAME policy_family_id: personal-vm policy_family_definition_overrides: '{"autotermination_minutes":{"type":"fixed","value":30}}' # Overrides as a multiline block-scalar JSON string (preserved verbatim). multiline: - name: policy_multiline + name: policy_multiline-$UNIQUE_NAME policy_family_id: personal-vm policy_family_definition_overrides: |- { @@ -23,7 +26,7 @@ resources: # Overrides as native YAML (normalized to compact JSON, numbers/booleans kept as JSON types). yaml: - name: policy_yaml + name: policy_yaml-$UNIQUE_NAME policy_family_id: personal-vm policy_family_definition_overrides: autotermination_minutes: diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt index 3f87e0b314b..3a0032161a8 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt @@ -2,28 +2,28 @@ >>> [CLI] bundle validate -o json { "json_string": { - "name": "policy_json_string", + "name": "policy_json_string-[UNIQUE_NAME]", "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30}}", "policy_family_id": "personal-vm" }, "multiline": { - "name": "policy_multiline", + "name": "policy_multiline-[UNIQUE_NAME]", "policy_family_definition_overrides": "{\n \"autotermination_minutes\": {\n \"type\": \"fixed\",\n \"value\": 30\n }\n}", "policy_family_id": "personal-vm" }, "yaml": { - "name": "policy_yaml", + "name": "policy_yaml-[UNIQUE_NAME]", "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true}}", "policy_family_id": "personal-vm" } } >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.json_string Created cluster_policies.multiline Created cluster_policies.yaml -Files: 4 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved) @@ -32,7 +32,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/policies/clusters/create", "body": { - "name": "policy_json_string", + "name": "policy_json_string-[UNIQUE_NAME]", "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30}}", "policy_family_id": "personal-vm" } @@ -41,7 +41,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/policies/clusters/create", "body": { - "name": "policy_multiline", + "name": "policy_multiline-[UNIQUE_NAME]", "policy_family_definition_overrides": "{\n \"autotermination_minutes\": {\n \"type\": \"fixed\",\n \"value\": 30\n }\n}", "policy_family_id": "personal-vm" } @@ -50,7 +50,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/policies/clusters/create", "body": { - "name": "policy_yaml", + "name": "policy_yaml-[UNIQUE_NAME]", "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true}}", "policy_family_id": "personal-vm" } @@ -62,6 +62,6 @@ The following resources will be deleted: delete resources.cluster_policies.multiline delete resources.cluster_policies.yaml -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 3 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script index fe1e5bd0507..03a77d152ad 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + trace $CLI bundle validate -o json | jq ".resources.cluster_policies" cleanup() { @@ -8,4 +10,4 @@ trap cleanup EXIT trace $CLI bundle deploy title "Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved)" -trace print_requests.py //policies/clusters/create --sort +trace print_requests.py //policies/clusters/create --sort \ No newline at end of file From 3cc0cb9074302bd136be0f8e9a470854e8aed95b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 24 Aug 2026 15:15:35 +0000 Subject: [PATCH 44/44] add trailing newline to cluster_policy test scripts Co-authored-by: Isaac --- .../resources/cluster_policies/backend_normalization/script | 2 +- acceptance/bundle/resources/cluster_policies/basic/script | 2 +- .../cluster_policies/definition_and_family_conflict/script | 2 +- .../resources/cluster_policies/definition_multiline/script | 2 +- .../bundle/resources/cluster_policies/definition_yaml/script | 2 +- .../bundle/resources/cluster_policies/out_of_band_change/script | 2 +- .../resources/cluster_policies/policy_family_definition/script | 2 +- .../resources/cluster_policies/policy_family_overrides/script | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/script b/acceptance/bundle/resources/cluster_policies/backend_normalization/script index 50f10926dd2..4100e517c44 100644 --- a/acceptance/bundle/resources/cluster_policies/backend_normalization/script +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/script @@ -18,4 +18,4 @@ done # yaml is normalized to compact JSON by the mutator, matching the string the backend # stores and returns, so it does not drift (plan below is a no-op). title "Plan after deploy: any drift means the backend reformatted a definition" -trace $CLI bundle plan \ No newline at end of file +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/cluster_policies/basic/script b/acceptance/bundle/resources/cluster_policies/basic/script index 82935208f7e..eff090af8ad 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/script +++ b/acceptance/bundle/resources/cluster_policies/basic/script @@ -36,4 +36,4 @@ trace $CLI bundle destroy --auto-approve title "Verify the destroy request" trace print_requests.py //policies/clusters/delete -trace $CLI bundle summary \ No newline at end of file +trace $CLI bundle summary diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script index 601a9520af1..4d83ab4b716 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script @@ -6,4 +6,4 @@ cleanup() { trap cleanup EXIT title "Deploying a policy with both definition and policy_family_id must fail\n" -musterr $CLI bundle deploy \ No newline at end of file +musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/script b/acceptance/bundle/resources/cluster_policies/definition_multiline/script index 8d0571f508f..35dd81e04fc 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_multiline/script +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/script @@ -8,4 +8,4 @@ title "Create body preserves the block-scalar definition as a newline-escaped st trace print_requests.py //policies/clusters trace $CLI bundle destroy --auto-approve -rm out.requests.txt \ No newline at end of file +rm out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/script b/acceptance/bundle/resources/cluster_policies/definition_yaml/script index b087604b4d6..92b030b5031 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/script +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/script @@ -8,4 +8,4 @@ title "Native YAML definition serializes to compact JSON, preserving numbers and trace print_requests.py //policies/clusters trace $CLI bundle destroy --auto-approve -rm out.requests.txt \ No newline at end of file +rm out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script index 1d7f9ed72c3..2f603a2b4fa 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script @@ -33,4 +33,4 @@ title "Verify the edit request restored the configured definition" trace print_requests.py //policies/clusters/edit title "Plan is a no-op again" -trace $CLI bundle plan \ No newline at end of file +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/script b/acceptance/bundle/resources/cluster_policies/policy_family_definition/script index 29c09952964..055fcaf6d26 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_definition/script +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/script @@ -18,4 +18,4 @@ trace $CLI bundle plan title "Redeploy issues no edit" rm -f out.requests.txt trace $CLI bundle deploy -trace print_requests.py //policies/clusters/edit \ No newline at end of file +trace print_requests.py //policies/clusters/edit diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script index 03a77d152ad..8f842f37b7d 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script @@ -10,4 +10,4 @@ trap cleanup EXIT trace $CLI bundle deploy title "Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved)" -trace print_requests.py //policies/clusters/create --sort \ No newline at end of file +trace print_requests.py //policies/clusters/create --sort