Add .workspace.URLID and fix review findings on .workspace

.nextchanges/bundles/workspace-template-urlid.md

@@ -0,0 +1 @@
+* Add `.workspace.URLID` to bundle templates: the workspace ID that `.workspace.URL` adds, or empty, so templates can use it without failing when the ID is unknown. ([#7388](https://github.com/databricks/cli/pull/7388))

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.protocol-relative.json

@@ -0,0 +1 @@
+{"path": "//other.test/jobs"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-w.json

@@ -0,0 +1 @@
+{"path": "/jobs?w=123"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-workspace-id.json

@@ -0,0 +1 @@
+{"path": "/jobs?workspace_id=123"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.supplied.json

@@ -0,0 +1 @@
+{"value": "set by user"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/output.txt

@@ -25,6 +25,7 @@ Hostname: [DATABRICKS_HOST]
 URL: [DATABRICKS_URL]
 URL "/jobs": [DATABRICKS_URL]/jobs
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list#top
+URLID: ""
 jobs_url: [DATABRICKS_URL]/jobs
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[DATABRICKS_URL]/jobs path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL] [WORKSPACE_ID] }]
@@ -55,6 +56,7 @@ Hostname: [DATABRICKS_HOST]
 URL: [DATABRICKS_URL]?w=[WORKSPACE_ID]
 URL "/jobs": [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 jobs_url: [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID]
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[DATABRICKS_URL]/jobs?w=[WORKSPACE_ID] path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL] [WORKSPACE_ID] [WORKSPACE_ID]}]
@@ -85,6 +87,7 @@ Hostname: [CUSTOM_HOSTNAME]
 URL: [CUSTOM_HOST]?w=[WORKSPACE_ID]
 URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 jobs_url: [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[CUSTOM_HOST]/jobs?w=[WORKSPACE_ID] path:/jobs?view=list#top workspace:{[CUSTOM_HOSTNAME] [CUSTOM_HOST] [WORKSPACE_ID] [WORKSPACE_ID]}]
@@ -115,6 +118,7 @@ Hostname: [DATABRICKS_HOST]
 URL: [DATABRICKS_URL]
 URL "/jobs": [DATABRICKS_URL]/jobs
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list#top
+URLID: ""
 jobs_url: [DATABRICKS_URL]/jobs
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[DATABRICKS_URL]/jobs path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL]  }]
@@ -145,20 +149,54 @@ Hostname: [CUSTOM_HOSTNAME]
 URL: [CUSTOM_HOST]?w=[WORKSPACE_ID]
 URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 URL "/pipelines?tab=all": [CUSTOM_HOST]/pipelines?tab=all&w=[WORKSPACE_ID]
+URLID: "[WORKSPACE_ID]"
 jobs_url: https://example.test/my-jobs
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:https://example.test/my-jobs path:/pipelines?tab=all workspace:{[CUSTOM_HOSTNAME] [CUSTOM_HOST] [WORKSPACE_ID] [WORKSPACE_ID]}]
 
+=== Failing default of an input with a value
+
+>>> [CLI] bundle debug render-template-schema ./supplied --profile no_workspace_id --input-file input.supplied.json
+{
+  "welcome_message": null,
+  "success_message": null,
+  "properties": {
+    "value": {
+      "description": "Value",
+      "default": "{{ .workspace.ID }}"
+    }
+  }
+}
+
+>>> [CLI] bundle init ./supplied --profile no_workspace_id --config-file input.supplied.json
+✨ Successfully initialized template
+value: set by user
+
+>>> musterr [CLI] bundle debug render-template-schema ./supplied --profile no_workspace_id
+Error: property value default: failed to render template "{{ .workspace.ID }}": template: base:1:13: executing "base" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle
+
 === Invalid URL paths
 
 >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.absolute-url.json
 Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
 Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "https://other.test/jobs" must be relative to the workspace host
 
+>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.protocol-relative.json
+Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
+Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "//other.test/jobs" must be relative to the workspace host
+
 >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.workspace-selector.json
 Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
 Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?o=123" must not set the "o" query parameter, the workspace is selected by the profile
 
+>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.selector-w.json
+Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
+Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?w=123" must not set the "w" query parameter, the workspace is selected by the profile
+
+>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.selector-workspace-id.json
+Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
+Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?workspace_id=123" must not set the "workspace_id" query parameter, the workspace is selected by the profile
+
 >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.semicolon.json
 Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
 Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: failed to parse query of workspace URL path "/jobs?w=999;x=1": invalid semicolon separator in query
@@ -171,6 +209,7 @@ Open [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top
 URL: [CUSTOM_HOST]?w=[WORKSPACE_ID]
 URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 
 === Workspace ID from the bundle in the working directory
 
@@ -180,6 +219,7 @@ Open [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top
 URL: [DATABRICKS_URL]?w=[WORKSPACE_ID]
 URL "/jobs": [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 
 === ID
 
@@ -192,6 +232,6 @@ ID: [WORKSPACE_ID]
 ID: [WORKSPACE_ID]
 
 >>> musterr [CLI] bundle init ./id --profile no_workspace_id
-Error: failed to compute file content for id.txt.tmpl. template: :1:17: executing "" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile or the DATABRICKS_WORKSPACE_ID environment variable
+Error: failed to compute file content for id.txt.tmpl. template: :1:17: executing "" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle
 
 >>> print_requests.py --get --oneline ^//telemetry-ext

acceptance/bundle/templates-machinery/workspace_variable/runtime/script

@@ -15,10 +15,20 @@ trace $CLI bundle init ./urls --profile workspace_custom_host --config-file inpu
 cat urls.txt
 rm urls.txt
 
+# A default that fails, here because the profile has no workspace ID, doesn't
+# matter when the input has a value: init doesn't use it, and the schema keeps
+# it as written. Without a value, the schema still fails.
+title "Failing default of an input with a value\n"
+trace $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id --input-file input.supplied.json | schema_fields
+trace $CLI bundle init ./supplied --profile no_workspace_id --config-file input.supplied.json
+cat value.txt
+rm value.txt
+trace musterr $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id
+
 title "Invalid URL paths\n"
-trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.absolute-url.json
-trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.workspace-selector.json
-trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.semicolon.json
+for input in absolute-url protocol-relative workspace-selector selector-w selector-workspace-id semicolon; do
+  trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.$input.json
+done
 
 # No profile, so that only the environment variable sets the ID.
 title "Workspace ID from environment variables\n"

acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/databricks_template_schema.json

@@ -0,0 +1,9 @@
+{
+    "properties": {
+        "value": {
+            "type": "string",
+            "description": "Value",
+            "default": "{{ .workspace.ID }}"
+        }
+    }
+}

acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/template/value.txt.tmpl

@@ -0,0 +1 @@
+value: {{ .value }}

acceptance/bundle/templates-machinery/workspace_variable/runtime/urls/template/urls.txt.tmpl

@@ -4,6 +4,7 @@ Hostname: {{ .workspace.Hostname }}
 URL: {{ .workspace.URL }}
 URL "/jobs": {{ .workspace.URL "/jobs" }}
 URL "{{ .path }}": {{ $ws.URL .path }}
+URLID: "{{ .workspace.URLID }}"
 jobs_url: {{ .jobs_url }}
 Root keys:{{ range $key, $value := . }} {{ $key }}{{ end }}
 Root data: {{ . }}

acceptance/bundle/templates-machinery/workspace_variable/script.prepare

@@ -1,9 +1,4 @@
-# The test server is reachable as both 127.0.0.1 and localhost. The second
-# stands in for a host shared by several workspaces.
-CUSTOM_HOST=${DATABRICKS_HOST/127.0.0.1/localhost}
-add_repl "$CUSTOM_HOST" CUSTOM_HOST
-add_repl "${CUSTOM_HOST#*://}" CUSTOM_HOSTNAME
-WORKSPACE_ID=$(curl -s "$DATABRICKS_HOST/.well-known/databricks-config" | jq -r .workspace_id)
+setup_custom_host
 
 # workspace_classic has no workspace_id, so the SDK resolves it from host
 # metadata. The "none" sentinel prevents that.

acceptance/script.prepare

@@ -115,6 +115,19 @@ add_repl() {
     env -u MSYS_NO_PATHCONV add_repl.py "$@"
 }
 
+# setup_custom_host sets CUSTOM_HOST to the test server reached as localhost
+# instead of 127.0.0.1, which stands in for a host shared by several workspaces,
+# and WORKSPACE_ID to the workspace ID that the test server reports. Call it
+# once per test, while DATABRICKS_HOST is still set: add_repl gives a repeated
+# replacement name a suffix, so a second call adds duplicate replacements.
+setup_custom_host() {
+    export CUSTOM_HOST=${DATABRICKS_HOST/127.0.0.1/localhost}
+    add_repl "$CUSTOM_HOST" CUSTOM_HOST
+    add_repl "${CUSTOM_HOST#*://}" CUSTOM_HOSTNAME
+    WORKSPACE_ID=$(curl -s "$DATABRICKS_HOST/.well-known/databricks-config" | jq -r .workspace_id)
+    export WORKSPACE_ID
+}
+
 nostamp() {
     # Same MSYS_NO_PATHCONV workaround as envsubst above: otherwise Python cannot
     # find nostamp.py on Windows.

libs/template/schema_renderer.go

@@ -135,6 +135,11 @@ func RenderSchema(ctx context.Context, reader Reader, input RenderSchemaInput) (
 			if def, ok := prop["default"].(string); ok {
 				rendered, err := renderString(def)
 				if err != nil {
+					// bundle init doesn't use the default of an input that has a value, so it
+					// can't fail there. Keep the default as written instead of failing here.
+					if supplied {
+						continue
+					}
 					return nil, fmt.Errorf("property %s default: %w", propName, err)
 				}
 				prop["default"] = rendered

libs/template/workspace_context.go

@@ -35,7 +35,8 @@ var userSetWorkspaceIDSources = []sdkconfig.SourceType{
 // it carries the workspace ID, which is required to address a workspace on a host shared by
 // several workspaces.
 //
-// The fields are strings so that templates which print the root data get stable output.
+// The fields are strings, not pointers, so that templates which print the root data with
+// {{ . }} get the values instead of addresses.
 type workspaceContext struct {
 	// Hostname is the workspace host without scheme, path, or query. It keeps the port if the
 	// host has one.
@@ -44,16 +45,22 @@ type workspaceContext struct {
 	host        string
 	workspaceID string
 
-	// urlWorkspaceID is the workspace ID that URL adds as ?w=. It is empty unless the user set the
-	// ID. Strict profile matching (DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH) compares the ID in the
-	// host with the profile's workspace_id, and a profile without workspace_id matches only a host
-	// without an ID.
-	urlWorkspaceID string
+	// URLID is the workspace ID that URL adds as ?w=. It is set when a workspace_id in the
+	// profile, DATABRICKS_WORKSPACE_ID, or the bundle's workspace_id sets the ID, and empty
+	// otherwise, so templates can check it without failing like ID does. It is empty for an ID
+	// that comes from ?o= in the profile's host or from host metadata. Strict profile matching
+	// (DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH) compares the bundle's workspace ID, which comes
+	// from ?w= in its host, with the profile's workspace_id, and a profile without workspace_id
+	// matches only a bundle without an ID.
+	//
+	// It comes last so that templates which print the root data keep their output.
+	URLID string
 }
 
-// loadWorkspaceContext returns the workspace context of the selected profile, or nil if the
-// schema declares a workspace input. A declared input owns the name even before it has a value.
-// Input values the schema doesn't declare don't claim the name, because bundle init ignores them.
+// loadWorkspaceContext returns the workspace context of the resolved configuration, or nil
+// if the schema declares a workspace input. A declared input owns the name even before it has
+// a value. Input values the schema doesn't declare don't claim the name, because bundle init
+// ignores them.
 func loadWorkspaceContext(ctx context.Context, schema *jsonschema.Schema) (*workspaceContext, error) {
 	if _, ok := schema.Properties[workspaceVariable]; ok {
 		return nil, nil
@@ -67,15 +74,15 @@ func loadWorkspaceContext(ctx context.Context, schema *jsonschema.Schema) (*work
 	if workspaceID == profile.WorkspaceIDNone {
 		workspaceID = ""
 	}
-	var urlWorkspaceID string
+	var urlID string
 	if attr, ok := cfg.GetAuthDetails().Configuration["workspace_id"]; ok && slices.Contains(userSetWorkspaceIDSources, attr.Source.Type) {
-		urlWorkspaceID = workspaceID
+		urlID = workspaceID
 	}
 	return &workspaceContext{
-		Hostname:       host.Host,
-		host:           cfg.Host,
-		workspaceID:    workspaceID,
-		urlWorkspaceID: urlWorkspaceID,
+		Hostname:    host.Host,
+		host:        cfg.Host,
+		workspaceID: workspaceID,
+		URLID:       urlID,
 	}, nil
 }
 
@@ -97,14 +104,13 @@ func templateData(values map[string]any, workspace *workspaceContext) map[string
 // shared by several workspaces. It doesn't make a request to discover the ID.
 func (c workspaceContext) ID() (string, error) {
 	if c.workspaceID == "" {
-		return "", errors.New("cannot determine workspace ID, please set workspace_id in the configuration profile or the DATABRICKS_WORKSPACE_ID environment variable")
+		return "", errors.New("cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle")
 	}
 	return c.workspaceID, nil
 }
 
 // URL returns the workspace URL with an optional path relative to the workspace host,
-// for example URL "/jobs". It adds ?w=<id> when the profile, an environment variable, or the
-// bundle sets the workspace ID.
+// for example URL "/jobs". It adds ?w=<id> when URLID is set.
 func (c workspaceContext) URL(path ...string) (string, error) {
 	if len(path) > 1 {
 		return "", fmt.Errorf("URL accepts at most one path, got %d", len(path))
@@ -113,11 +119,11 @@ func (c workspaceContext) URL(path ...string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to parse workspace host %q: %w", c.host, err)
 	}
-	// Unlike workspaceurls.WorkspaceBaseURL, this adds ?w= also when the hostname has the ID
-	// (adb-<id>), because strict profile matching reads the ID only from the query.
-	if c.urlWorkspaceID != "" {
+	// The workspaceurls package leaves out ?w= when the hostname has the ID (adb-<id>). This
+	// adds it, because strict profile matching doesn't read the ID from the hostname.
+	if c.URLID != "" {
 		query := base.Query()
-		query.Set("w", c.urlWorkspaceID)
+		query.Set("w", c.URLID)
 		base.RawQuery = query.Encode()
 	}
 	if len(path) == 0 {

libs/template/workspace_context_test.go

@@ -87,7 +87,7 @@ func TestPromptForValuesWithWorkspace(t *testing.T) {
 // Acceptance tests can't use an adb-<id> hostname, because the test server is reachable only
 // on localhost.
 func TestWorkspaceURLKeepsIDOnHostnameWithID(t *testing.T) {
-	ws := workspaceContext{host: "https://adb-111.4.azuredatabricks.net", workspaceID: "111", urlWorkspaceID: "111"}
+	ws := workspaceContext{host: "https://adb-111.4.azuredatabricks.net", workspaceID: "111", URLID: "111"}
 
 	// Strict profile matching reads the ID from the query, not from the hostname.
 	host, err := ws.URL()