🤖 fix: a CoderProvisioner replaces only keys it recorded and binds key records to the control plane UID

api/v1alpha1/coderprovisioner_types.go

@@ -36,6 +36,13 @@ const (
 	// operator deletes no key for such an entry. The message names each
 	// entry, and the operator removes the condition when none is left.
 	CoderProvisionerConditionUnverifiedKeyKept = "UnverifiedKeyKept"
+	// CoderProvisionerConditionControlPlaneChangedKeyKept is True while
+	// status.pendingKeys holds an entry whose control plane has another UID
+	// than the entry records, or an entry without a UID whose coderURL is not
+	// the current Coder API URL of its control plane. The operator sends no
+	// operator token for such an entry. The message names each entry, and the
+	// operator removes the condition when none is left.
+	CoderProvisionerConditionControlPlaneChangedKeyKept = "ControlPlaneChangedKeyKept"
 
 	// DefaultProvisionerKeySecretKey is the default data key for provisioner key secrets.
 	DefaultProvisionerKeySecretKey = "key"
@@ -110,6 +117,15 @@ type CoderProvisionerStatus struct {
 	ControlPlaneRefName string `json:"controlPlaneRefName,omitempty"`
 	// ControlPlaneURL is the control plane URL last applied to the provisioner key.
 	ControlPlaneURL string `json:"controlPlaneURL,omitempty"`
+	// ControlPlaneUID is the UID of the control plane last applied to the
+	// provisioner key. The operator sends the operator token of the control
+	// plane with controlPlaneRefName for this key only while that control
+	// plane has this UID, and rotates the key when it has another UID. When it
+	// is empty, it sends that token only to the current Coder API URL of that
+	// control plane, and the next reconcile looks the key up there by
+	// organizationID and provisionerKeyID: it sets this field when Coder holds
+	// the key, and creates a new key when Coder answers that it has none.
+	ControlPlaneUID string `json:"controlPlaneUID,omitempty"`
 	// SecretRef references the provisioner key secret data currently in use.
 	SecretRef *SecretKeySelector `json:"secretRef,omitempty"`
 	// PendingKeys lists the provisioner keys that can exist in Coder besides
@@ -129,6 +145,15 @@ type CoderProvisionerKeyRecord struct {
 	// ControlPlaneRefName is the CoderControlPlane whose operator token the
 	// operator used for the key.
 	ControlPlaneRefName string `json:"controlPlaneRefName"`
+	// ControlPlaneUID is the UID of that CoderControlPlane. The operator
+	// sends its operator token for this entry only while the control plane
+	// with controlPlaneRefName has this UID. When it is empty, it sends that
+	// token only when coderURL is the current Coder API URL of that control
+	// plane. Otherwise it keeps the entry and sets the condition
+	// ControlPlaneChangedKeyKept. When a rotation replaces the key of status
+	// without status.controlPlaneUID, the entry of that key records the UID
+	// while coderURL is the current Coder API URL of that control plane.
+	ControlPlaneUID string `json:"controlPlaneUID,omitempty"`
 	// CoderURL is the Coder API URL that the operator sent the key request to.
 	CoderURL string `json:"coderURL"`
 	// OrganizationName is the Coder organization of the key.

config/crd/bases/coder.com_coderprovisioners.yaml

@@ -417,6 +417,17 @@ spec:
                 description: ControlPlaneRefName is the control plane ref name last
                   applied to the provisioner key.
                 type: string
+              controlPlaneUID:
+                description: |-
+                  ControlPlaneUID is the UID of the control plane last applied to the
+                  provisioner key. The operator sends the operator token of the control
+                  plane with controlPlaneRefName for this key only while that control
+                  plane has this UID, and rotates the key when it has another UID. When it
+                  is empty, it sends that token only to the current Coder API URL of that
+                  control plane, and the next reconcile looks the key up there by
+                  organizationID and provisionerKeyID: it sets this field when Coder holds
+                  the key, and creates a new key when Coder answers that it has none.
+                type: string
               controlPlaneURL:
                 description: ControlPlaneURL is the control plane URL last applied
                   to the provisioner key.
@@ -457,6 +468,17 @@ spec:
                         ControlPlaneRefName is the CoderControlPlane whose operator token the
                         operator used for the key.
                       type: string
+                    controlPlaneUID:
+                      description: |-
+                        ControlPlaneUID is the UID of that CoderControlPlane. The operator
+                        sends its operator token for this entry only while the control plane
+                        with controlPlaneRefName has this UID. When it is empty, it sends that
+                        token only when coderURL is the current Coder API URL of that control
+                        plane. Otherwise it keeps the entry and sets the condition
+                        ControlPlaneChangedKeyKept. When a rotation replaces the key of status
+                        without status.controlPlaneUID, the entry of that key records the UID
+                        while coderURL is the current Coder API URL of that control plane.
+                      type: string
                     keyID:
                       description: KeyID is the provisioner key ID that Coder returned.
                       type: string

dist/install.yaml

@@ -6185,6 +6185,17 @@ spec:
                 description: ControlPlaneRefName is the control plane ref name last
                   applied to the provisioner key.
                 type: string
+              controlPlaneUID:
+                description: |-
+                  ControlPlaneUID is the UID of the control plane last applied to the
+                  provisioner key. The operator sends the operator token of the control
+                  plane with controlPlaneRefName for this key only while that control
+                  plane has this UID, and rotates the key when it has another UID. When it
+                  is empty, it sends that token only to the current Coder API URL of that
+                  control plane, and the next reconcile looks the key up there by
+                  organizationID and provisionerKeyID: it sets this field when Coder holds
+                  the key, and creates a new key when Coder answers that it has none.
+                type: string
               controlPlaneURL:
                 description: ControlPlaneURL is the control plane URL last applied
                   to the provisioner key.
@@ -6225,6 +6236,17 @@ spec:
                         ControlPlaneRefName is the CoderControlPlane whose operator token the
                         operator used for the key.
                       type: string
+                    controlPlaneUID:
+                      description: |-
+                        ControlPlaneUID is the UID of that CoderControlPlane. The operator
+                        sends its operator token for this entry only while the control plane
+                        with controlPlaneRefName has this UID. When it is empty, it sends that
+                        token only when coderURL is the current Coder API URL of that control
+                        plane. Otherwise it keeps the entry and sets the condition
+                        ControlPlaneChangedKeyKept. When a rotation replaces the key of status
+                        without status.controlPlaneUID, the entry of that key records the UID
+                        while coderURL is the current Coder API URL of that control plane.
+                      type: string
                     keyID:
                       description: KeyID is the provisioner key ID that Coder returned.
                       type: string

docs/reference/api/coderprovisioner.md

@@ -40,6 +40,7 @@
 | `tagsHash` | string | TagsHash is a deterministic hash of spec.tags last applied to the provisioner key. |
 | `controlPlaneRefName` | string | ControlPlaneRefName is the control plane ref name last applied to the provisioner key. |
 | `controlPlaneURL` | string | ControlPlaneURL is the control plane URL last applied to the provisioner key. |
+| `controlPlaneUID` | string | ControlPlaneUID is the UID of the control plane last applied to the provisioner key. The operator sends the operator token of the control plane with controlPlaneRefName for this key only while that control plane has this UID, and rotates the key when it has another UID. When it is empty, it sends that token only to the current Coder API URL of that control plane, and the next reconcile looks the key up there by organizationID and provisionerKeyID: it sets this field when Coder holds the key, and creates a new key when Coder answers that it has none. |
 | `secretRef` | [SecretKeySelector](#secretkeyselector) | SecretRef references the provisioner key secret data currently in use. |
 | `pendingKeys` | [CoderProvisionerKeyRecord](#coderprovisionerkeyrecord) array | PendingKeys lists the provisioner keys that can exist in Coder besides the key in provisionerKeyID: a key that the operator is about to create or has not yet written to the provisioner key Secret, and replaced keys whose deletion Coder has not confirmed. The operator deletes the key of each entry with organizationID and keyID at its coderURL with the operator token of its control plane, and removes the entry when Coder confirms that the key is gone. |
 
@@ -53,6 +54,7 @@ or was about to create, at one Coder endpoint.
 | Field | Type | Description |
 | --- | --- | --- |
 | `controlPlaneRefName` | string | ControlPlaneRefName is the CoderControlPlane whose operator token the operator used for the key. |
+| `controlPlaneUID` | string | ControlPlaneUID is the UID of that CoderControlPlane. The operator sends its operator token for this entry only while the control plane with controlPlaneRefName has this UID. When it is empty, it sends that token only when coderURL is the current Coder API URL of that control plane. Otherwise it keeps the entry and sets the condition ControlPlaneChangedKeyKept. When a rotation replaces the key of status without status.controlPlaneUID, the entry of that key records the UID while coderURL is the current Coder API URL of that control plane. |
 | `coderURL` | string | CoderURL is the Coder API URL that the operator sent the key request to. |
 | `organizationName` | string | OrganizationName is the Coder organization of the key. |
 | `keyName` | string | KeyName is the provisioner key name in Coder. |

docs/tutorials/deploy-coderprovisioner.md

@@ -54,6 +54,8 @@ Expected result: the phase is `Ready`, and the conditions include `CredentialsRe
 
 The operator creates the provisioner key Secret and owns it. If you set `spec.key.secretName`, name a Secret that does not exist yet: the operator refuses a Secret that it does not own (reason `KeySecretNotOwned`).
 
+Each provisioner needs its own key name in its Coder organization. The key name is `spec.key.name`, or the provisioner name when `spec.key.name` is empty. Before the operator creates or replaces a key, it looks up the key name in Coder. When Coder holds a key with that name, and the status of this provisioner does not record its ID, the operator does not delete or replace that key, because another `CoderProvisioner` (for example one with the same name in another namespace) or a person can use it. It keeps the current key, Secret and Deployment of this provisioner, and only deletes the replaced keys that `status.pendingKeys` records for earlier targets: `ProvisionerKeyReady` is `False` with reason `KeyNameInUse`, and the message names the key and its ID. Set `spec.key.name` to a name that nothing else uses, or delete that key in Coder if nothing uses it. The operator checks again every 2 minutes. The same happens when the operator lost the answer to a key create, because status then has no ID of the new key, and for status without key records, for example after a restore without status.
+
 If you add `spec.extraEnv` entries with a `secretKeyRef`, each Secret needs the label `coder.com/credential=true` and the annotation `coder.com/credential-destinations` with the origin of the control plane `status.url`. Otherwise `CredentialsReady` is `False` and the operator does not change the Deployment. See [Configure credential Secrets for proxies and provisioners](../how-to/workspace-proxy-credentials.md).
 
 The operator creates these resources in the `coder` namespace:
@@ -80,7 +82,7 @@ kubectl delete coderprovisioner coderprovisioner-sample -n coder
 
 The operator then deletes the provisioner keys in Coder that this provisioner created. It finds the key in the provisioner key Secret by the organization and key IDs in `status.organizationID` and `status.provisionerKeyID`, not by the names in the spec. It also deletes the key of each entry in `status.pendingKeys` that has an `organizationID` and a `keyID`. The entries name keys that the operator created but did not write to the Secret, for example because a reconcile failed, and keys that a rotation replaced but that Coder did not confirm as deleted. Each entry names the Coder URL of the key and the control plane whose operator token the operator uses to delete it. An entry without these IDs names a key that the operator started to create before Coder returned its ID. For such an entry, the operator deletes no key, because another provisioner or a person can use the same key name in that organization. It removes the entry when Coder has no key with that name, or when that key is the key in `status.provisionerKeyID`. Otherwise, and while the operator cannot reach Coder for that check, the entry stays, the condition `UnverifiedKeyKept` is `True` and names the entry, and deletion of the provisioner leaves the key in Coder. If no other provisioner or person uses the key, delete it with `coder provisioner keys delete <name> --org <organization>`. The operator checks again every 2 minutes, and removes the entry and the condition once the key is gone. When the status has no key, for example because the provisioner never passed its checks, the operator removes its finalizer without a call to Coder.
 
-The same records apply while the provisioner runs. When `spec.controlPlaneRef`, the control plane URL, the organization, the key name, or the tags change, the operator creates a new key and deletes the replaced keys, each at its own Coder URL. When a rotation fails after Coder created the new key, the next reconcile writes another new key to the Secret, also when you reverted the change. An entry stays in `status.pendingKeys` until Coder confirms that its key is gone, and the operator tries again every 2 minutes. An entry whose control plane was deleted stays for good: delete its key by hand in Coder, then remove the entry, for example the first one with `kubectl patch coderprovisioner coderprovisioner-sample -n coder --subresource=status --type=json -p '[{"op":"remove","path":"/status/pendingKeys/0"}]'`.
+The same records apply while the provisioner runs. When `spec.controlPlaneRef`, the control plane URL, the organization, the key name, or the tags change, the operator creates a new key and deletes the replaced keys by their recorded organization and key IDs, each at its own Coder URL, so a renamed Coder organization does not hide a replaced key. When a rotation fails after Coder created the new key, the next reconcile writes another new key to the Secret, also when you reverted the change. An entry stays in `status.pendingKeys` until Coder confirms that its key is gone, and the operator tries again every 2 minutes. Each entry and `status.controlPlaneUID` also record the UID of the control plane. The operator sends the operator token of a control plane for a record only while the control plane has the recorded UID. When a control plane is deleted and created again under the same name, the new one can run another Coder backend, also at the same URL. The operator then creates a new key for the provisioner, and sends the new token for no key of the old control plane. Such an entry stays, the condition `ControlPlaneChangedKeyKept` is `True` and names it, and deletion of the provisioner leaves that key in Coder. A record without a UID, from an operator version that did not record it, counts as bound to its control plane only at the current URL of that control plane. For status without `status.controlPlaneUID`, the next reconcile looks the applied key up at the current control plane by `status.organizationID` and `status.provisionerKeyID`, so a renamed organization does not hide it. When Coder holds the key there, the operator records the UID of that control plane. When Coder answers that it has no such key, the operator creates a new key. When a change of the spec or the control plane replaces the key of such status, its entry in `status.pendingKeys` records the UID of its control plane while that control plane has the recorded URL. An entry whose control plane was deleted, or that `ControlPlaneChangedKeyKept` names, stays for good: delete its key by hand in Coder, then remove the entry, for example the first one with `kubectl patch coderprovisioner coderprovisioner-sample -n coder --subresource=status --type=json -p '[{"op":"remove","path":"/status/pendingKeys/0"}]'`.
 
 This cleanup is best-effort. When the operator cannot reach Coder, for example because the control plane or its operator token is gone, or when the delete call fails, the operator still removes its finalizer, and the key stays valid in Coder. Delete the provisioner while the control plane runs. If cleanup could not reach Coder, list the keys with `coder provisioner keys list --org <organization>`, and delete the key by hand with `coder provisioner keys delete <name> --org <organization>`.
 

internal/coderbootstrap/client.go

@@ -94,8 +94,8 @@ type RegisterWorkspaceProxyResponse struct {
 type Client interface {
 	EnsureWorkspaceProxy(context.Context, RegisterWorkspaceProxyRequest) (RegisterWorkspaceProxyResponse, error)
 	EnsureProvisionerKey(context.Context, EnsureProvisionerKeyRequest) (EnsureProvisionerKeyResponse, error)
-	DeleteProvisionerKey(ctx context.Context, coderURL, sessionToken, orgName, keyName string) error
 	DeleteProvisionerKeyByID(context.Context, DeleteProvisionerKeyByIDRequest) error
+	ProvisionerKeyByID(context.Context, ProvisionerKeyByIDRequest) (*codersdk.ProvisionerKey, error)
 	ProvisionerKeyByName(context.Context, ProvisionerKeyByNameRequest) (*codersdk.ProvisionerKey, error)
 	Entitlements(ctx context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error)
 }