From 0a640a844660de18d9ac308bc87c2efe8d16f3bf Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 18 Nov 2026 16:03:21 +0000 Subject: [PATCH] Stop per-image metric series accumulating and validate intervals A later audit pass found that the image_lookup_duration and image_failures_total series are keyed by the raw image from the pod spec and were never replaced when a live pod's image changed. RemoveImage also tried to delete them with a container_type label those two metrics do not have, which matches nothing, so they were only removed with the pod. Keep one duration series per container, restart the failure counter when the image changes, and delete both in RemoveImage using labels they do have. Also: - reject --image-cache-timeout and --kube-interval values of zero or less, which panic time.NewTicker or disable the cache; - use the request context in /readyz instead of context.Background(); - give the dl.k8s.io version fetch a 10 second per-request timeout. Signed-off-by: David Collom --- cmd/app/app.go | 7 +++++-- cmd/app/options.go | 9 +++++++++ cmd/app/options_test.go | 7 +++++++ pkg/controller/kube_controller.go | 1 + pkg/metrics/helpers.go | 6 ++++++ pkg/metrics/metrics.go | 33 +++++++++++++++++++++++++++---- pkg/metrics/metrics_test.go | 22 +++++++++++++++++++++ 7 files changed, 79 insertions(+), 6 deletions(-) diff --git a/cmd/app/app.go b/cmd/app/app.go index e539dad1..a438560c 100644 --- a/cmd/app/app.go +++ b/cmd/app/app.go @@ -41,6 +41,9 @@ func NewCommand(ctx context.Context) *cobra.Command { Long: helpOutput, RunE: func(_ *cobra.Command, _ []string) error { opts.complete() + if err := opts.validateIntervals(); err != nil { + return err + } logLevel, err := logrus.ParseLevel(opts.LogLevel) if err != nil { @@ -94,8 +97,8 @@ func NewCommand(ctx context.Context) *cobra.Command { // Readiness probe if err := mgr.AddMetricsServerExtraHandler("/readyz", - http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if mgr.GetCache().WaitForCacheSync(context.Background()) { + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mgr.GetCache().WaitForCacheSync(r.Context()) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ready")) } else { diff --git a/cmd/app/options.go b/cmd/app/options.go index 9aa5f0c4..2790cda9 100644 --- a/cmd/app/options.go +++ b/cmd/app/options.go @@ -476,6 +476,15 @@ func (o *Options) assignSelfhosted(envs []string) { } } +// validateIntervals rejects durations that would panic time.NewTicker or +// disable the cache. +func (o *Options) validateIntervals() error { + if o.CacheTimeout <= 0 || o.KubeInterval <= 0 { + return fmt.Errorf("--image-cache-timeout and --kube-interval must be greater than zero") + } + return nil +} + func validSelfHostedOpts(opts *Options) bool { // opts set using env vars if opts.Client.Selfhosted != nil { diff --git a/cmd/app/options_test.go b/cmd/app/options_test.go index d8a3c04a..06eaa3ec 100644 --- a/cmd/app/options_test.go +++ b/cmd/app/options_test.go @@ -3,6 +3,7 @@ package app import ( "os" "testing" + "time" "github.com/stretchr/testify/assert" @@ -359,3 +360,9 @@ func TestAssignSelfhosted(t *testing.T) { }) } } + +func TestValidateIntervals(t *testing.T) { + assert.NoError(t, (&Options{CacheTimeout: time.Minute, KubeInterval: time.Minute}).validateIntervals()) + assert.Error(t, (&Options{CacheTimeout: 0, KubeInterval: time.Minute}).validateIntervals()) + assert.Error(t, (&Options{CacheTimeout: time.Minute, KubeInterval: -1}).validateIntervals()) +} diff --git a/pkg/controller/kube_controller.go b/pkg/controller/kube_controller.go index 3ff726d1..e1f39e6d 100644 --- a/pkg/controller/kube_controller.go +++ b/pkg/controller/kube_controller.go @@ -48,6 +48,7 @@ func NewKubeReconciler( log = log.WithField("controller", "channel") httpClient := retryablehttp.NewClient() + httpClient.HTTPClient.Timeout = 10 * time.Second httpClient.RetryMax = 3 httpClient.RetryWaitMin = 1 * time.Second httpClient.RetryWaitMax = 30 * time.Second diff --git a/pkg/metrics/helpers.go b/pkg/metrics/helpers.go index ea3e2ff9..5aa637ff 100644 --- a/pkg/metrics/helpers.go +++ b/pkg/metrics/helpers.go @@ -31,6 +31,12 @@ func buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL s } } +// buildContainerImageLabels matches the metrics that are keyed by container +// and image but have no container_type label. +func buildContainerImageLabels(namespace, pod, container string) prometheus.Labels { + return prometheus.Labels{"namespace": namespace, "pod": pod, "container": container} +} + func buildPodPartialLabels(namespace, pod string) prometheus.Labels { return prometheus.Labels{ "namespace": namespace, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d37e958f..fe449ce0 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "strings" "sync" "time" @@ -35,6 +36,11 @@ type Metrics struct { cache k8sclient.Reader + // errorImage remembers which image each container's failure counter is + // for (key "namespace/pod/container"), so the counter of a replaced image + // can be dropped instead of accumulating series. + errorImage map[string]string + // Contains all metrics for the roundtripper roundTripper *RoundTripper @@ -118,8 +124,9 @@ func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, cache k8sclient.R ) return &Metrics{ - log: log.WithField("module", "metrics"), - cache: cache, + log: log.WithField("module", "metrics"), + cache: cache, + errorImage: map[string]string{}, registry: reg, containerImageVersion: containerImageVersion, @@ -168,9 +175,12 @@ func (m *Metrics) RemoveImage(namespace, pod, container, containerType string) { labels := buildContainerPartialLabels(namespace, pod, container, containerType) total += m.containerImageVersion.DeletePartialMatch(labels) - total += m.containerImageDuration.DeletePartialMatch(labels) total += m.containerImageChecked.DeletePartialMatch(labels) - total += m.containerImageErrors.DeletePartialMatch(labels) + // Duration and errors have no container_type label, so a partial match + // that includes it would match nothing. + total += m.containerImageDuration.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) + total += m.containerImageErrors.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) + delete(m.errorImage, namespace+"/"+pod+"/"+container) total += m.containerImageTimestamp.DeletePartialMatch(labels) total += m.containerImageAvailable.DeletePartialMatch(labels) @@ -194,6 +204,11 @@ func (m *Metrics) RemovePod(namespace, pod string) { total += m.containerImageErrors.DeletePartialMatch( buildPodPartialLabels(namespace, pod), ) + for k := range m.errorImage { + if strings.HasPrefix(k, namespace+"/"+pod+"/") { + delete(m.errorImage, k) + } + } total += m.containerImageTimestamp.DeletePartialMatch( buildPodPartialLabels(namespace, pod), ) @@ -213,6 +228,8 @@ func (m *Metrics) RegisterImageDuration(namespace, pod, container, image string, return } + // One series per container: drop the one for a previous image. + m.containerImageDuration.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) m.containerImageDuration.WithLabelValues( namespace, pod, container, image, ).Set(time.Since(startTime).Seconds()) @@ -227,6 +244,14 @@ func (m *Metrics) ReportError(namespace, pod, container, imageURL string) { return } + // A replaced image starts a new failure count rather than a new series + // next to the old one. + key := namespace + "/" + pod + "/" + container + if prev, ok := m.errorImage[key]; ok && prev != imageURL { + m.containerImageErrors.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) + } + m.errorImage[key] = imageURL + m.containerImageErrors.WithLabelValues( namespace, pod, container, imageURL, ).Inc() diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index a7847a7c..9bc621e4 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -278,3 +278,25 @@ func TestPodAnnotationsChangeAfterRegistration(t *testing.T) { testutil.CollectAndCount(metrics.containerImageVersion.MetricVec, MetricNamespace+"_is_latest_version"), ) } + +// A container whose image changes, or that is no longer checked, must not +// leave lookup duration and failure series behind. +func TestImageSeriesDoNotAccumulate(t *testing.T) { + m := New(logrus.NewEntry(logrus.New()), prometheus.NewRegistry(), fakek8s) + ctx := context.Background() + require.NoError(t, fakek8s.DeleteAllOf(ctx, &corev1.Pod{})) + require.NoError(t, fakek8s.Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns"}, + })) + + for _, img := range []string{"a", "b", "c"} { + m.RegisterImageDuration("ns", "p", "c", img, time.Now()) + m.ReportError("ns", "p", "c", img) + } + assert.Equal(t, 1, testutil.CollectAndCount(m.containerImageDuration)) + assert.Equal(t, 1, testutil.CollectAndCount(m.containerImageErrors)) + + m.RemoveImage("ns", "p", "c", "init") + assert.Equal(t, 0, testutil.CollectAndCount(m.containerImageDuration)) + assert.Equal(t, 0, testutil.CollectAndCount(m.containerImageErrors)) +}