diff --git a/.nextchanges/bundles/grant-privilege-enum-no-validate.md b/.nextchanges/bundles/grant-privilege-enum-no-validate.md new file mode 100644 index 00000000000..ede26200eee --- /dev/null +++ b/.nextchanges/bundles/grant-privilege-enum-no-validate.md @@ -0,0 +1 @@ +* Accept grant `privileges` values that the SDK enum does not list yet: `bundle validate` no longer warns (or errors under `--strict`) on them, and the PyDABs `Privilege` enum accepts unknown strings instead of raising. The JSON schema and PyDABs still advertise the known values for completion. ([#7315](https://github.com/databricks/cli/pull/7315)) diff --git a/acceptance/bundle/validate/enum/databricks.yml b/acceptance/bundle/validate/enum/databricks.yml index 8bff766951c..0f6759ec9a8 100644 --- a/acceptance/bundle/validate/enum/databricks.yml +++ b/acceptance/bundle/validate/enum/databricks.yml @@ -57,3 +57,14 @@ resources: ebs_volume_type: "INVALID_VOLUME_TYPE" node_type_id: "i3.xlarge" num_workers: 1 + + schemas: + my_schema: + name: my_schema + catalog_name: main + grants: + - principal: user@databricks.com + privileges: + # Grant privileges are not validated against the enum, so this + # does not trigger a warning (the SDK enum lags the platform). + - INVALID_PRIVILEGE diff --git a/bundle/internal/validation/enum.go b/bundle/internal/validation/enum.go index df58193c898..7662153af91 100644 --- a/bundle/internal/validation/enum.go +++ b/bundle/internal/validation/enum.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" "github.com/databricks/cli/libs/structs/structwalk" + "github.com/databricks/databricks-sdk-go/service/catalog" ) type EnumPatternInfo struct { @@ -126,6 +127,14 @@ func extractEnumFields(typ reflect.Type) ([]EnumPatternInfo, error) { } } + // The platform accepts grant privileges that the SDK's catalog.Privilege + // enum does not yet know about, so validating against it rejects values the + // platform allows (see https://github.com/databricks/cli/issues/6770). The + // JSON schema still advertises the enum; we just don't warn/error on it here. + if fieldType == reflect.TypeFor[catalog.Privilege]() { + return false + } + // Check if this field type is an enum if isEnumType(fieldType) { // Get enum values diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index f57e5cbd781..5ee12ac6a36 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -40,8 +40,6 @@ var EnumFields = map[string][]string{ "resources.apps.*.resources[*].uc_securable.permission": {"EXECUTE", "MODIFY", "READ_VOLUME", "SELECT", "USE_CONNECTION", "WRITE_VOLUME"}, "resources.apps.*.resources[*].uc_securable.securable_type": {"CONNECTION", "FUNCTION", "MODEL_SERVICE", "TABLE", "VOLUME"}, - "resources.catalogs.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, - "resources.cluster_policies.*.permissions[*].level": {"CAN_USE"}, "resources.clusters.*.aws_attributes.availability": {"ON_DEMAND", "SPOT", "SPOT_WITH_FALLBACK"}, @@ -64,7 +62,6 @@ var EnumFields = map[string][]string{ "resources.experiments.*.permissions[*].level": {"CAN_EDIT", "CAN_MANAGE", "CAN_READ"}, "resources.external_locations.*.encryption_details.sse_encryption_details.algorithm": {"AWS_SSE_KMS", "AWS_SSE_S3"}, - "resources.external_locations.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.features.*.function.function_type": {"APPROX_COUNT_DISTINCT", "APPROX_PERCENTILE", "AVG", "COUNT", "FIRST", "FUNCTION_TYPE_UNSPECIFIED", "LAST", "MAX", "MIN", "STDDEV_POP", "STDDEV_SAMP", "SUM", "VAR_POP", "VAR_SAMP"}, "resources.features.*.source.request_source.flat_schema.fields[*].data_type": {"BINARY", "BOOLEAN", "DATE", "DECIMAL", "DOUBLE", "FLOAT", "INTEGER", "LONG", "SHORT", "STRING", "TIMESTAMP"}, @@ -177,18 +174,15 @@ var EnumFields = map[string][]string{ "resources.mcp_services.*.config.rate_limits[*].key": {"RATE_LIMIT_KEY_SERVICE", "RATE_LIMIT_KEY_SERVICE_PRINCIPAL", "RATE_LIMIT_KEY_USER", "RATE_LIMIT_KEY_USER_DEFAULT", "RATE_LIMIT_KEY_USER_GROUP"}, "resources.mcp_services.*.config.rate_limits[*].renewal_period": {"RATE_LIMIT_RENEWAL_PERIOD_HOUR", "RATE_LIMIT_RENEWAL_PERIOD_MINUTE"}, - "resources.mcp_services.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.model_provider_services.*.config.provider_type": {"EXTERNAL_MODEL_PROVIDER_TYPE_AMAZON_BEDROCK", "EXTERNAL_MODEL_PROVIDER_TYPE_ANTHROPIC", "EXTERNAL_MODEL_PROVIDER_TYPE_AZURE_OPENAI", "EXTERNAL_MODEL_PROVIDER_TYPE_BEDROCK_MANTLE", "EXTERNAL_MODEL_PROVIDER_TYPE_CUSTOM", "EXTERNAL_MODEL_PROVIDER_TYPE_GEMINI_ENTERPRISE", "EXTERNAL_MODEL_PROVIDER_TYPE_MICROSOFT_FOUNDRY", "EXTERNAL_MODEL_PROVIDER_TYPE_OPENAI"}, "resources.model_provider_services.*.config.rate_limits[*].key": {"RATE_LIMIT_KEY_SERVICE", "RATE_LIMIT_KEY_SERVICE_PRINCIPAL", "RATE_LIMIT_KEY_USER", "RATE_LIMIT_KEY_USER_DEFAULT", "RATE_LIMIT_KEY_USER_GROUP"}, "resources.model_provider_services.*.config.rate_limits[*].renewal_period": {"RATE_LIMIT_RENEWAL_PERIOD_HOUR", "RATE_LIMIT_RENEWAL_PERIOD_MINUTE"}, - "resources.model_provider_services.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.model_services.*.config.rate_limits[*].key": {"RATE_LIMIT_KEY_SERVICE", "RATE_LIMIT_KEY_SERVICE_PRINCIPAL", "RATE_LIMIT_KEY_USER", "RATE_LIMIT_KEY_USER_DEFAULT", "RATE_LIMIT_KEY_USER_GROUP"}, "resources.model_services.*.config.rate_limits[*].renewal_period": {"RATE_LIMIT_RENEWAL_PERIOD_HOUR", "RATE_LIMIT_RENEWAL_PERIOD_MINUTE"}, "resources.model_services.*.config.routing.destinations[*].destination_type": {"DESTINATION_TYPE_EXTERNAL_FOUNDATION_MODEL", "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", "DESTINATION_TYPE_PROVISIONED_THROUGHPUT_FOUNDATION_MODEL"}, "resources.model_services.*.config.routing.fallback.destinations[*].destination_type": {"DESTINATION_TYPE_EXTERNAL_FOUNDATION_MODEL", "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", "DESTINATION_TYPE_PROVISIONED_THROUGHPUT_FOUNDATION_MODEL"}, - "resources.model_services.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.model_serving_endpoints.*.ai_gateway.guardrails.input.pii.behavior": {"BLOCK", "MASK", "NONE"}, "resources.model_serving_endpoints.*.ai_gateway.guardrails.output.pii.behavior": {"BLOCK", "MASK", "NONE"}, @@ -297,15 +291,9 @@ var EnumFields = map[string][]string{ "resources.quality_monitors.*.inference_log.problem_type": {"PROBLEM_TYPE_CLASSIFICATION", "PROBLEM_TYPE_REGRESSION"}, "resources.quality_monitors.*.schedule.pause_status": {"PAUSED", "UNPAUSED", "UNSPECIFIED"}, - "resources.registered_models.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, - - "resources.schemas.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, - "resources.secret_scopes.*.backend_type": {"AZURE_KEYVAULT", "DATABRICKS"}, "resources.secret_scopes.*.permissions[*].level": {"READ", "WRITE", "MANAGE"}, - "resources.secrets.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, - "resources.sql_warehouses.*.channel.name": {"CHANNEL_NAME_CURRENT", "CHANNEL_NAME_CUSTOM", "CHANNEL_NAME_PREVIEW", "CHANNEL_NAME_PREVIOUS"}, "resources.sql_warehouses.*.permissions[*].level": {"CAN_MANAGE", "CAN_MONITOR", "CAN_USE", "CAN_VIEW", "IS_OWNER"}, "resources.sql_warehouses.*.spot_instance_policy": {"COST_OPTIMIZED", "POLICY_UNSPECIFIED", "RELIABILITY_OPTIMIZED"}, @@ -323,12 +311,10 @@ var EnumFields = map[string][]string{ "resources.vector_search_endpoints.*.permissions[*].level": {"CAN_CREATE", "CAN_MANAGE", "CAN_USE"}, "resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_type": {"CONTINUOUS", "TRIGGERED"}, - "resources.vector_search_indexes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.vector_search_indexes.*.index_subtype": {"FULL_TEXT", "HYBRID", "VECTOR"}, "resources.vector_search_indexes.*.index_type": {"DELTA_SYNC", "DIRECT_ACCESS"}, - "resources.volumes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FEATURE", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FEATURE", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, - "resources.volumes.*.volume_type": {"EXTERNAL", "MANAGED"}, + "resources.volumes.*.volume_type": {"EXTERNAL", "MANAGED"}, "variables.*.type": {"complex"}, diff --git a/python/codegen/codegen/generated_enum.py b/python/codegen/codegen/generated_enum.py index a9a551a26a1..fc86fb2cbf1 100644 --- a/python/codegen/codegen/generated_enum.py +++ b/python/codegen/codegen/generated_enum.py @@ -7,6 +7,10 @@ from codegen.generated_dataclass import _append_description from codegen.jsonschema import Schema, is_experimental_stage +# Enums whose platform value set grows faster than the schema. They keep their +# members for completion, but unknown strings are accepted instead of rejected. +OPEN_ENUMS = {"catalog.Privilege"} + @dataclass(kw_only=True) class GeneratedEnum: @@ -16,6 +20,7 @@ class GeneratedEnum: description: Optional[str] experimental: bool deprecated: bool + open: bool = False def generate_enum(namespace: str, schema_name: str, schema: Schema) -> GeneratedEnum: @@ -43,6 +48,7 @@ def generate_enum(namespace: str, schema_name: str, schema: Schema) -> Generated description=schema.description, experimental=is_experimental_stage(schema.stage), deprecated=schema.deprecated or False, + open=schema_name in OPEN_ENUMS, ) @@ -71,6 +77,10 @@ def get_code(generated: GeneratedEnum) -> str: b.indent().append(f'{key} = "{value}"') b.newline() + if generated.open: + b.newline() + _append_missing(b, generated.class_name) + b.newline() # Example: @@ -80,11 +90,41 @@ def get_code(generated: GeneratedEnum) -> str: b.append(generated.class_name).append('Param = Literal["') b.append_list(list(generated.values.values()), sep='", "') b.append('"] | ', generated.class_name) + + if generated.open: + b.append(" | str") + b.newline() return b.build() +def _append_missing(b: CodeBuilder, class_name: str): + # Example: + # + # @classmethod + # def _missing_(cls, value: object) -> "Color | None": + # ... + # + # Unknown strings become pseudo-members, the same way stdlib Flag builds + # them. They are cached in _value2member_map_ so repeated lookups return + # the same object and dataclass equality keeps working. + b.indent().append("@classmethod").newline() + b.indent().append( + f'def _missing_(cls, value: object) -> "{class_name} | None":' + ).newline() + b.indent().indent().append("if not isinstance(value, str):").newline() + b.indent().indent().indent().append("return None").newline() + b.newline() + b.indent().indent().append("member = object.__new__(cls)").newline() + b.indent().indent().append("member._name_ = value").newline() + b.indent().indent().append("member._value_ = value").newline() + b.newline() + b.indent().indent().append( + "return cls._value2member_map_.setdefault(value, member) # type:ignore" + ).newline() + + def _camel_to_upper_snake(value): s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", value) s1 = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1) diff --git a/python/codegen/codegen/main.py b/python/codegen/codegen/main.py index 83735b8dc4c..f0e37857a70 100644 --- a/python/codegen/codegen/main.py +++ b/python/codegen/codegen/main.py @@ -23,6 +23,9 @@ def main(output: str): schemas = openapi.get_schemas() + if missing := generated_enum.OPEN_ENUMS - schemas.keys(): + raise ValueError(f"Cannot mark unknown schemas as open enums: {missing}") + schemas = openapi_patch.add_extra_required_fields(schemas) schemas = openapi_patch.remove_unsupported_fields(schemas) schemas = openapi_patch.drop_field_descriptions(schemas) diff --git a/python/codegen/codegen_tests/test_generated_enum.py b/python/codegen/codegen_tests/test_generated_enum.py index 300b021e6ad..edd50f26122 100644 --- a/python/codegen/codegen_tests/test_generated_enum.py +++ b/python/codegen/codegen_tests/test_generated_enum.py @@ -1,4 +1,4 @@ -from codegen.generated_enum import GeneratedEnum, generate_enum +from codegen.generated_enum import GeneratedEnum, generate_enum, get_code from codegen.jsonschema import Schema, SchemaType @@ -20,4 +20,48 @@ def test_generate_enum(): description="enum description", experimental=False, deprecated=False, + open=False, ) + + +def test_generate_open_enum(): + generated = generate_enum( + namespace="catalogs", + schema_name="catalog.Privilege", + schema=Schema(enum=["SELECT"], type=SchemaType.STRING), + ) + + assert generated.open + + +def test_get_code_open_enum(): + code = get_code( + GeneratedEnum( + class_name="Privilege", + package="databricks.bundles.catalogs._models.privilege", + values={"SELECT": "SELECT"}, + description=None, + experimental=False, + deprecated=False, + open=True, + ) + ) + + assert "def _missing_(cls, value: object)" in code + assert 'PrivilegeParam = Literal["SELECT"] | Privilege | str' in code + + +def test_get_code_closed_enum(): + code = get_code( + GeneratedEnum( + class_name="Color", + package="databricks.bundles.bananas._models.color", + values={"RED": "RED"}, + description=None, + experimental=False, + deprecated=False, + ) + ) + + assert "_missing_" not in code + assert 'ColorParam = Literal["RED"] | Color\n' in code diff --git a/python/databricks/bundles/catalogs/_models/privilege.py b/python/databricks/bundles/catalogs/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/catalogs/_models/privilege.py +++ b/python/databricks/bundles/catalogs/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/external_locations/_models/privilege.py b/python/databricks/bundles/external_locations/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/external_locations/_models/privilege.py +++ b/python/databricks/bundles/external_locations/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/mcp_services/_models/privilege.py b/python/databricks/bundles/mcp_services/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/mcp_services/_models/privilege.py +++ b/python/databricks/bundles/mcp_services/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/model_provider_services/_models/privilege.py b/python/databricks/bundles/model_provider_services/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/model_provider_services/_models/privilege.py +++ b/python/databricks/bundles/model_provider_services/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/model_services/_models/privilege.py b/python/databricks/bundles/model_services/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/model_services/_models/privilege.py +++ b/python/databricks/bundles/model_services/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/registered_models/_models/privilege.py b/python/databricks/bundles/registered_models/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/registered_models/_models/privilege.py +++ b/python/databricks/bundles/registered_models/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/schemas/_models/privilege.py b/python/databricks/bundles/schemas/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/schemas/_models/privilege.py +++ b/python/databricks/bundles/schemas/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/secrets/_models/privilege.py b/python/databricks/bundles/secrets/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/secrets/_models/privilege.py +++ b/python/databricks/bundles/secrets/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/vector_search_indexes/_models/privilege.py b/python/databricks/bundles/vector_search_indexes/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/vector_search_indexes/_models/privilege.py +++ b/python/databricks/bundles/vector_search_indexes/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks/bundles/volumes/_models/privilege.py b/python/databricks/bundles/volumes/_models/privilege.py index 470bedfe94c..382f3917d5a 100644 --- a/python/databricks/bundles/volumes/_models/privilege.py +++ b/python/databricks/bundles/volumes/_models/privilege.py @@ -60,6 +60,17 @@ class Privilege(Enum): READ_FEATURE = "READ_FEATURE" EXTERNAL_USE_LOCATION = "EXTERNAL_USE_LOCATION" + @classmethod + def _missing_(cls, value: object) -> "Privilege | None": + if not isinstance(value, str): + return None + + member = object.__new__(cls) + member._name_ = value + member._value_ = value + + return cls._value2member_map_.setdefault(value, member) # type:ignore + PrivilegeParam = ( Literal[ @@ -119,4 +130,5 @@ class Privilege(Enum): "EXTERNAL_USE_LOCATION", ] | Privilege + | str ) diff --git a/python/databricks_tests/core/test_open_enum.py b/python/databricks_tests/core/test_open_enum.py new file mode 100644 index 00000000000..aa4eec089f4 --- /dev/null +++ b/python/databricks_tests/core/test_open_enum.py @@ -0,0 +1,41 @@ +"""Privilege is generated as an open enum: unknown values are accepted instead of raising.""" + +import pytest + +from databricks.bundles.schemas import Privilege, PrivilegeAssignment + + +def test_known_value(): + assert Privilege("SELECT") is Privilege.SELECT + + +def test_unknown_value_is_accepted(): + privilege = Privilege("CREATE_SECRET") + + assert isinstance(privilege, Privilege) + assert privilege.value == "CREATE_SECRET" + assert privilege.name == "CREATE_SECRET" + + +def test_unknown_value_is_cached(): + assert Privilege("CREATE_FEATURE") is Privilege("CREATE_FEATURE") + + +def test_unknown_non_string_raises(): + with pytest.raises(ValueError): + Privilege(42) + + +def test_unknown_value_round_trip(): + assignment = PrivilegeAssignment.from_dict( + {"principal": "user@databricks.com", "privileges": ["CREATE_SECRET", "SELECT"]} + ) + + assert assignment == PrivilegeAssignment( + principal="user@databricks.com", + privileges=[Privilege("CREATE_SECRET"), Privilege.SELECT], + ) + assert assignment.as_dict() == { + "principal": "user@databricks.com", + "privileges": ["CREATE_SECRET", "SELECT"], + }