🤖 fix: revoke a CoderProvisioner key created at an intermediate control plane endpoint

api/v1alpha1/coderprovisioner_types.go

@@ -29,6 +29,13 @@ const (
 	// referenced credential Secret and setting passed the operator's checks.
 	// When it is False, the operator keeps the last applied Deployment.
 	CoderProvisionerConditionCredentialsReady = "CredentialsReady"
+	// CoderProvisionerConditionUnverifiedKeyKept is True while
+	// status.pendingKeys holds an entry without key IDs that the operator
+	// could not remove: Coder holds a key with its name, which nothing shows
+	// this provisioner created, or the operator could not check Coder. The
+	// operator deletes no key for such an entry. The message names each
+	// entry, and the operator removes the condition when none is left.
+	CoderProvisionerConditionUnverifiedKeyKept = "UnverifiedKeyKept"
 
 	// DefaultProvisionerKeySecretKey is the default data key for provisioner key secrets.
 	DefaultProvisionerKeySecretKey = "key"
@@ -105,6 +112,42 @@ type CoderProvisionerStatus struct {
 	ControlPlaneURL string `json:"controlPlaneURL,omitempty"`
 	// SecretRef references the provisioner key secret data currently in use.
 	SecretRef *SecretKeySelector `json:"secretRef,omitempty"`
+	// PendingKeys lists the provisioner keys that can exist in Coder besides
+	// the key in provisionerKeyID: a key that the operator is about to create
+	// or has not yet written to the provisioner key Secret, and replaced keys
+	// whose deletion Coder has not confirmed. The operator deletes the key of
+	// each entry with organizationID and keyID at its coderURL with the
+	// operator token of its control plane, and removes the entry when Coder
+	// confirms that the key is gone.
+	// +listType=atomic
+	PendingKeys []CoderProvisionerKeyRecord `json:"pendingKeys,omitempty"`
+}
+
+// CoderProvisionerKeyRecord names a provisioner key that the operator created,
+// or was about to create, at one Coder endpoint.
+type CoderProvisionerKeyRecord struct {
+	// ControlPlaneRefName is the CoderControlPlane whose operator token the
+	// operator used for the key.
+	ControlPlaneRefName string `json:"controlPlaneRefName"`
+	// CoderURL is the Coder API URL that the operator sent the key request to.
+	CoderURL string `json:"coderURL"`
+	// OrganizationName is the Coder organization of the key.
+	OrganizationName string `json:"organizationName"`
+	// KeyName is the provisioner key name in Coder.
+	KeyName string `json:"keyName"`
+	// OrganizationID is the organization ID that Coder returned for the key.
+	// When organizationID or keyID is empty, the operator deletes no key. It
+	// removes the entry when Coder has no key with keyName, or when that key
+	// is the key in provisionerKeyID. Otherwise it keeps the entry and sets
+	// the condition UnverifiedKeyKept.
+	OrganizationID string `json:"organizationID,omitempty"`
+	// KeyID is the provisioner key ID that Coder returned.
+	KeyID string `json:"keyID,omitempty"`
+	// Unpublished is true from before the create until the operator records
+	// a key as applied. While it is true, the Secret can hold this key, or a
+	// key that a rotation deleted, so the operator replaces the key in the
+	// Secret.
+	Unpublished bool `json:"unpublished,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

config/crd/bases/coder.com_coderprovisioners.yaml

@@ -434,6 +434,62 @@ spec:
                 description: OrganizationName is the organization name last applied
                   to the provisioner key.
                 type: string
+              pendingKeys:
+                description: |-
+                  PendingKeys lists the provisioner keys that can exist in Coder besides
+                  the key in provisionerKeyID: a key that the operator is about to create
+                  or has not yet written to the provisioner key Secret, and replaced keys
+                  whose deletion Coder has not confirmed. The operator deletes the key of
+                  each entry with organizationID and keyID at its coderURL with the
+                  operator token of its control plane, and removes the entry when Coder
+                  confirms that the key is gone.
+                items:
+                  description: |-
+                    CoderProvisionerKeyRecord names a provisioner key that the operator created,
+                    or was about to create, at one Coder endpoint.
+                  properties:
+                    coderURL:
+                      description: CoderURL is the Coder API URL that the operator
+                        sent the key request to.
+                      type: string
+                    controlPlaneRefName:
+                      description: |-
+                        ControlPlaneRefName is the CoderControlPlane whose operator token the
+                        operator used for the key.
+                      type: string
+                    keyID:
+                      description: KeyID is the provisioner key ID that Coder returned.
+                      type: string
+                    keyName:
+                      description: KeyName is the provisioner key name in Coder.
+                      type: string
+                    organizationID:
+                      description: |-
+                        OrganizationID is the organization ID that Coder returned for the key.
+                        When organizationID or keyID is empty, the operator deletes no key. It
+                        removes the entry when Coder has no key with keyName, or when that key
+                        is the key in provisionerKeyID. Otherwise it keeps the entry and sets
+                        the condition UnverifiedKeyKept.
+                      type: string
+                    organizationName:
+                      description: OrganizationName is the Coder organization of the
+                        key.
+                      type: string
+                    unpublished:
+                      description: |-
+                        Unpublished is true from before the create until the operator records
+                        a key as applied. While it is true, the Secret can hold this key, or a
+                        key that a rotation deleted, so the operator replaces the key in the
+                        Secret.
+                      type: boolean
+                  required:
+                  - coderURL
+                  - controlPlaneRefName
+                  - keyName
+                  - organizationName
+                  type: object
+                type: array
+                x-kubernetes-list-type: atomic
               phase:
                 description: Phase is a high-level readiness indicator.
                 type: string

dist/install.yaml

@@ -6190,6 +6190,62 @@ spec:
                 description: OrganizationName is the organization name last applied
                   to the provisioner key.
                 type: string
+              pendingKeys:
+                description: |-
+                  PendingKeys lists the provisioner keys that can exist in Coder besides
+                  the key in provisionerKeyID: a key that the operator is about to create
+                  or has not yet written to the provisioner key Secret, and replaced keys
+                  whose deletion Coder has not confirmed. The operator deletes the key of
+                  each entry with organizationID and keyID at its coderURL with the
+                  operator token of its control plane, and removes the entry when Coder
+                  confirms that the key is gone.
+                items:
+                  description: |-
+                    CoderProvisionerKeyRecord names a provisioner key that the operator created,
+                    or was about to create, at one Coder endpoint.
+                  properties:
+                    coderURL:
+                      description: CoderURL is the Coder API URL that the operator
+                        sent the key request to.
+                      type: string
+                    controlPlaneRefName:
+                      description: |-
+                        ControlPlaneRefName is the CoderControlPlane whose operator token the
+                        operator used for the key.
+                      type: string
+                    keyID:
+                      description: KeyID is the provisioner key ID that Coder returned.
+                      type: string
+                    keyName:
+                      description: KeyName is the provisioner key name in Coder.
+                      type: string
+                    organizationID:
+                      description: |-
+                        OrganizationID is the organization ID that Coder returned for the key.
+                        When organizationID or keyID is empty, the operator deletes no key. It
+                        removes the entry when Coder has no key with keyName, or when that key
+                        is the key in provisionerKeyID. Otherwise it keeps the entry and sets
+                        the condition UnverifiedKeyKept.
+                      type: string
+                    organizationName:
+                      description: OrganizationName is the Coder organization of the
+                        key.
+                      type: string
+                    unpublished:
+                      description: |-
+                        Unpublished is true from before the create until the operator records
+                        a key as applied. While it is true, the Secret can hold this key, or a
+                        key that a rotation deleted, so the operator replaces the key in the
+                        Secret.
+                      type: boolean
+                  required:
+                  - coderURL
+                  - controlPlaneRefName
+                  - keyName
+                  - organizationName
+                  type: object
+                type: array
+                x-kubernetes-list-type: atomic
               phase:
                 description: Phase is a high-level readiness indicator.
                 type: string

docs/reference/api/coderprovisioner.md

@@ -41,9 +41,25 @@
 | `controlPlaneRefName` | string | ControlPlaneRefName is the control plane ref name last applied to the provisioner key. |
 | `controlPlaneURL` | string | ControlPlaneURL is the control plane URL last applied to the provisioner key. |
 | `secretRef` | [SecretKeySelector](#secretkeyselector) | SecretRef references the provisioner key secret data currently in use. |
+| `pendingKeys` | [CoderProvisionerKeyRecord](#coderprovisionerkeyrecord) array | PendingKeys lists the provisioner keys that can exist in Coder besides the key in provisionerKeyID: a key that the operator is about to create or has not yet written to the provisioner key Secret, and replaced keys whose deletion Coder has not confirmed. The operator deletes the key of each entry with organizationID and keyID at its coderURL with the operator token of its control plane, and removes the entry when Coder confirms that the key is gone. |
 
 ## Referenced types
 
+### CoderProvisionerKeyRecord
+
+CoderProvisionerKeyRecord names a provisioner key that the operator created,
+or was about to create, at one Coder endpoint.
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `controlPlaneRefName` | string | ControlPlaneRefName is the CoderControlPlane whose operator token the operator used for the key. |
+| `coderURL` | string | CoderURL is the Coder API URL that the operator sent the key request to. |
+| `organizationName` | string | OrganizationName is the Coder organization of the key. |
+| `keyName` | string | KeyName is the provisioner key name in Coder. |
+| `organizationID` | string | OrganizationID is the organization ID that Coder returned for the key. When organizationID or keyID is empty, the operator deletes no key. It removes the entry when Coder has no key with keyName, or when that key is the key in provisionerKeyID. Otherwise it keeps the entry and sets the condition UnverifiedKeyKept. |
+| `keyID` | string | KeyID is the provisioner key ID that Coder returned. |
+| `unpublished` | boolean | Unpublished is true from before the create until the operator records a key as applied. While it is true, the Secret can hold this key, or a key that a rotation deleted, so the operator replaces the key in the Secret. |
+
 ### CoderProvisionerKeySpec
 
 CoderProvisionerKeySpec configures provisioner key naming and storage.

docs/tutorials/deploy-coderprovisioner.md

@@ -78,7 +78,9 @@ kubectl get secret coderprovisioner-sample-provisioner-key -n coder
 kubectl delete coderprovisioner coderprovisioner-sample -n coder
 ```
 
-The operator then deletes the provisioner key in Coder that this provisioner created. It finds the key by the organization and key IDs in `status.organizationID` and `status.provisionerKeyID`, not by the names in the spec. When the status has no key, for example because the provisioner never passed its checks, the operator removes its finalizer without a call to Coder.
+The operator then deletes the provisioner keys in Coder that this provisioner created. It finds the key in the provisioner key Secret by the organization and key IDs in `status.organizationID` and `status.provisionerKeyID`, not by the names in the spec. It also deletes the key of each entry in `status.pendingKeys` that has an `organizationID` and a `keyID`. The entries name keys that the operator created but did not write to the Secret, for example because a reconcile failed, and keys that a rotation replaced but that Coder did not confirm as deleted. Each entry names the Coder URL of the key and the control plane whose operator token the operator uses to delete it. An entry without these IDs names a key that the operator started to create before Coder returned its ID. For such an entry, the operator deletes no key, because another provisioner or a person can use the same key name in that organization. It removes the entry when Coder has no key with that name, or when that key is the key in `status.provisionerKeyID`. Otherwise, and while the operator cannot reach Coder for that check, the entry stays, the condition `UnverifiedKeyKept` is `True` and names the entry, and deletion of the provisioner leaves the key in Coder. If no other provisioner or person uses the key, delete it with `coder provisioner keys delete <name> --org <organization>`. The operator checks again every 2 minutes, and removes the entry and the condition once the key is gone. When the status has no key, for example because the provisioner never passed its checks, the operator removes its finalizer without a call to Coder.
+
+The same records apply while the provisioner runs. When `spec.controlPlaneRef`, the control plane URL, the organization, the key name, or the tags change, the operator creates a new key and deletes the replaced keys, each at its own Coder URL. When a rotation fails after Coder created the new key, the next reconcile writes another new key to the Secret, also when you reverted the change. An entry stays in `status.pendingKeys` until Coder confirms that its key is gone, and the operator tries again every 2 minutes. An entry whose control plane was deleted stays for good: delete its key by hand in Coder, then remove the entry, for example the first one with `kubectl patch coderprovisioner coderprovisioner-sample -n coder --subresource=status --type=json -p '[{"op":"remove","path":"/status/pendingKeys/0"}]'`.
 
 This cleanup is best-effort. When the operator cannot reach Coder, for example because the control plane or its operator token is gone, or when the delete call fails, the operator still removes its finalizer, and the key stays valid in Coder. Delete the provisioner while the control plane runs. If cleanup could not reach Coder, list the keys with `coder provisioner keys list --org <organization>`, and delete the key by hand with `coder provisioner keys delete <name> --org <organization>`.
 

internal/coderbootstrap/client.go

@@ -96,6 +96,7 @@ type Client interface {
 	EnsureProvisionerKey(context.Context, EnsureProvisionerKeyRequest) (EnsureProvisionerKeyResponse, error)
 	DeleteProvisionerKey(ctx context.Context, coderURL, sessionToken, orgName, keyName string) error
 	DeleteProvisionerKeyByID(context.Context, DeleteProvisionerKeyByIDRequest) error
+	ProvisionerKeyByName(context.Context, ProvisionerKeyByNameRequest) (*codersdk.ProvisionerKey, error)
 	Entitlements(ctx context.Context, coderURL, sessionToken string) (codersdk.Entitlements, error)
 }
 

internal/coderbootstrap/provisionerkeys.go

@@ -1,6 +1,7 @@
 package coderbootstrap
 
 import (
+	"cmp"
 	"context"
 	"errors"
 	"net/http"
@@ -140,6 +141,50 @@ func (c *SDKClient) DeleteProvisionerKey(ctx context.Context, coderURL, sessionT
 	return xerrors.Errorf("delete provisioner key %q: %w", keyName, err)
 }
 
+// ProvisionerKeyByNameRequest identifies a provisioner key by its name.
+type ProvisionerKeyByNameRequest struct {
+	CoderURL         string
+	SessionToken     string
+	OrganizationName string
+	KeyName          string
+}
+
+// ProvisionerKeyByName returns the provisioner key named req.KeyName in the
+// organization req.OrganizationName. It returns nil when Coder answers that
+// the organization or a key with that name does not exist.
+func (c *SDKClient) ProvisionerKeyByName(ctx context.Context, req ProvisionerKeyByNameRequest) (*codersdk.ProvisionerKey, error) {
+	if err := validateProvisionerKeyInputs(req.CoderURL, req.SessionToken, req.KeyName); err != nil {
+		return nil, err
+	}
+
+	client, err := newAuthenticatedClient(req.CoderURL, req.SessionToken)
+	if err != nil {
+		return nil, err
+	}
+
+	organization, err := resolveOrganizationByName(ctx, client, cmp.Or(req.OrganizationName, codersdk.DefaultOrganization))
+	if isNotFound(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	key, err := withOptionalRateLimitBypass(ctx, func(requestCtx context.Context) (*codersdk.ProvisionerKey, error) {
+		return findOrganizationProvisionerKey(requestCtx, client, organization.ID, req.KeyName)
+	})
+	if isNotFound(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if key != nil && key.ID == uuid.Nil {
+		return nil, xerrors.Errorf("assertion failed: provisioner key %q returned an empty ID", req.KeyName)
+	}
+
+	return key, nil
+}
+
 // DeleteProvisionerKeyByIDRequest identifies one provisioner key by the IDs
 // that the caller recorded when it applied the key.
 type DeleteProvisionerKeyByIDRequest struct {
@@ -208,9 +253,15 @@ func (c *SDKClient) DeleteProvisionerKeyByID(ctx context.Context, req DeleteProv
 	return nil
 }
 
+// coderNotFoundMessage is the message of the 404 that Coder answers for a
+// missing organization or provisioner key (coderd/httpapi/httpapi.go:149 and
+// coderd/httpmw/provisionerkey.go:45-46 in v2.37.3). Another 404, for example
+// from a proxy or for a wrong base URL, does not prove that the key is gone.
+const coderNotFoundMessage = "Resource not found or you do not have access to this resource"
+
 func isNotFound(err error) bool {
 	var apiErr *codersdk.Error
-	return errors.As(err, &apiErr) && apiErr.StatusCode() == http.StatusNotFound
+	return errors.As(err, &apiErr) && apiErr.StatusCode() == http.StatusNotFound && apiErr.Message == coderNotFoundMessage
 }
 
 // Entitlements returns deployment entitlements for the given coderd instance.