diff --git a/.nextchanges/bundles/secret-parent-rename.md b/.nextchanges/bundles/secret-parent-rename.md new file mode 100644 index 00000000000..0d3c9b7d1ce --- /dev/null +++ b/.nextchanges/bundles/secret-parent-rename.md @@ -0,0 +1 @@ +* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7381](https://github.com/databricks/cli/pull/7381)) diff --git a/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl new file mode 100644 index 00000000000..98e310a0dc4 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + secrets: + secret1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} + comment: c1 diff --git a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt new file mode 100644 index 00000000000..5ba571ac7c1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -0,0 +1,440 @@ + +=== Deploy a catalog with a schema and a secret in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test_catalog_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema", + "value": "secret-value" + } +} + +=== Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "skip", + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated secrets.secret1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== Rename the catalog and change the secret's comment: the update reaches the secret at its new path +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "full_name": "test_catalog_[UNIQUE_NAME].myschema2", + "name": "myschema2", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema2", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" + }, + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema2.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Updated secrets.secret1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c2", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== The state tracks the new path and the plan is a no-op +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "secrets": { + "secret1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/secrets/parent_rename/script b/acceptance/bundle/resources/secrets/parent_rename/script new file mode 100644 index 00000000000..8c5fa6cc7c9 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/script @@ -0,0 +1,39 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +} + +title "Deploy a catalog with a schema and a secret in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema.mysecret" | jq -r .create_time)" SECRET_CREATE_TIME + +title "Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request" +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "Rename the catalog and change the secret's comment: the update reaches the secret at its new path" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "comment: c1" "comment: c2" +plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_renamed_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "The state tracks the new path and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/parent_rename/test.toml b/acceptance/bundle/resources/secrets/parent_rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 75c91cdc4d5..c0d57151fd5 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -473,8 +473,10 @@ func (a *Adapter) validate() error { if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { return errors.New("resourceConfig has updatable_id_fields but DoUpdateWithID is not implemented") } - if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger { - return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields") + // A parent's rename also re-keys a resource through its provided_id_fields (see followParentRenames). + canUpdateWithID := hasUpdateWithIDTrigger || (a.resourceConfig != nil && len(a.resourceConfig.ProvidedIDFields) > 0) + if a.doUpdateWithID != nil && !canUpdateWithID { + return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields and provided_id_fields") } } diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 22ca3d7f49f..82b5ace212c 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -1,3 +1,12 @@ +provided_id_fields: + # The ID is the full name. UC cannot rename a secret, but it moves the secret with a renamed + # catalog or schema, which re-keys it (see followParentRenames). + - field: catalog_name + reason: id_field + - field: schema_name + reason: id_field + - field: name + reason: id_field sensitive_fields: - field: value - field: effective_value diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 2db0a334f82..2a1235de14f 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -104,6 +104,17 @@ func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *catalog return remapSecretRemote(response), nil } +// DoUpdateWithID re-keys the secret after its catalog or schema is renamed: UC moves the secret +// with its parent. A secret cannot be renamed itself, so this is the only way its ID changes. +func (r *ResourceSecret) DoUpdateWithID(ctx context.Context, _ string, state *catalog.Secret, entry *PlanEntry) (string, *catalog.Secret, error) { + newID := state.CatalogName + "." + state.SchemaName + "." + state.Name + if entry.Changes.OnlyParentRenamed() { + return newID, nil, nil + } + remote, err := r.DoUpdate(ctx, newID, state, entry) + return newID, remote, err +} + // DoDelete deletes the secret. func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *catalog.Secret) error { return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 27ee4f53102..6e9ed132551 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -133,7 +133,7 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { } } -// moveCatalogChildren mirrors UC, where the schemas, volumes and registered models of a renamed catalog move with it. +// moveCatalogChildren mirrors UC, where the schemas, volumes, registered models and secrets of a renamed catalog move with it. func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName string) { for key, schema := range s.Schemas { if schema.CatalogName != oldCatalogName { @@ -160,9 +160,18 @@ func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName strin model.CatalogName = newCatalogName s.moveRegisteredModel(key, model) } + for key, secret := range s.UCSecrets { + if secret.CatalogName != oldCatalogName { + continue + } + delete(s.UCSecrets, key) + secret.CatalogName = newCatalogName + secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name + s.UCSecrets[secret.FullName] = secret + } } -// CatalogsDelete mirrors UC, where deleting a catalog with force deletes its schemas, volumes and registered models too. +// CatalogsDelete mirrors UC, where deleting a catalog with force deletes its schemas, volumes, registered models and secrets too. func (s *FakeWorkspace) CatalogsDelete(req Request, name string) Response { defer s.LockUnlock()() @@ -187,6 +196,11 @@ func (s *FakeWorkspace) CatalogsDelete(req Request, name string) Response { delete(s.RegisteredModelVersions, key) } } + for key, secret := range s.UCSecrets { + if secret.CatalogName == name { + delete(s.UCSecrets, key) + } + } } return Response{} } diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 0f77cbe5640..78fdd13903f 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -116,7 +116,7 @@ func (s *FakeWorkspace) SchemasUpdate(req Request, name string) Response { } } -// moveSchemaChildren mirrors UC, where the volumes and registered models of a renamed schema move with it. +// moveSchemaChildren mirrors UC, where the volumes, registered models and secrets of a renamed schema move with it. func (s *FakeWorkspace) moveSchemaChildren(catalogName, oldSchemaName, newSchemaName string) { for key, volume := range s.Volumes { if volume.CatalogName != catalogName || volume.SchemaName != oldSchemaName { @@ -134,9 +134,18 @@ func (s *FakeWorkspace) moveSchemaChildren(catalogName, oldSchemaName, newSchema model.SchemaName = newSchemaName s.moveRegisteredModel(key, model) } + for key, secret := range s.UCSecrets { + if secret.CatalogName != catalogName || secret.SchemaName != oldSchemaName { + continue + } + delete(s.UCSecrets, key) + secret.SchemaName = newSchemaName + secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name + s.UCSecrets[secret.FullName] = secret + } } -// SchemasDelete mirrors UC, where deleting a schema with force deletes the volumes and registered models in it too. +// SchemasDelete mirrors UC, where deleting a schema with force deletes the volumes, registered models and secrets in it too. func (s *FakeWorkspace) SchemasDelete(req Request, fullName string) Response { defer s.LockUnlock()() @@ -157,6 +166,11 @@ func (s *FakeWorkspace) SchemasDelete(req Request, fullName string) Response { delete(s.RegisteredModelVersions, key) } } + for key, secret := range s.UCSecrets { + if secret.CatalogName == schema.CatalogName && secret.SchemaName == schema.Name { + delete(s.UCSecrets, key) + } + } } return Response{} }