diff --git a/cmd/app/app.go b/cmd/app/app.go index e539dad..a438560 100644 --- a/cmd/app/app.go +++ b/cmd/app/app.go @@ -41,6 +41,9 @@ func NewCommand(ctx context.Context) *cobra.Command { Long: helpOutput, RunE: func(_ *cobra.Command, _ []string) error { opts.complete() + if err := opts.validateIntervals(); err != nil { + return err + } logLevel, err := logrus.ParseLevel(opts.LogLevel) if err != nil { @@ -94,8 +97,8 @@ func NewCommand(ctx context.Context) *cobra.Command { // Readiness probe if err := mgr.AddMetricsServerExtraHandler("/readyz", - http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if mgr.GetCache().WaitForCacheSync(context.Background()) { + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mgr.GetCache().WaitForCacheSync(r.Context()) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ready")) } else { diff --git a/cmd/app/options.go b/cmd/app/options.go index 9aa5f0c..2790cda 100644 --- a/cmd/app/options.go +++ b/cmd/app/options.go @@ -476,6 +476,15 @@ func (o *Options) assignSelfhosted(envs []string) { } } +// validateIntervals rejects durations that would panic time.NewTicker or +// disable the cache. +func (o *Options) validateIntervals() error { + if o.CacheTimeout <= 0 || o.KubeInterval <= 0 { + return fmt.Errorf("--image-cache-timeout and --kube-interval must be greater than zero") + } + return nil +} + func validSelfHostedOpts(opts *Options) bool { // opts set using env vars if opts.Client.Selfhosted != nil { diff --git a/cmd/app/options_test.go b/cmd/app/options_test.go index d8a3c04..06eaa3e 100644 --- a/cmd/app/options_test.go +++ b/cmd/app/options_test.go @@ -3,6 +3,7 @@ package app import ( "os" "testing" + "time" "github.com/stretchr/testify/assert" @@ -359,3 +360,9 @@ func TestAssignSelfhosted(t *testing.T) { }) } } + +func TestValidateIntervals(t *testing.T) { + assert.NoError(t, (&Options{CacheTimeout: time.Minute, KubeInterval: time.Minute}).validateIntervals()) + assert.Error(t, (&Options{CacheTimeout: 0, KubeInterval: time.Minute}).validateIntervals()) + assert.Error(t, (&Options{CacheTimeout: time.Minute, KubeInterval: -1}).validateIntervals()) +} diff --git a/pkg/controller/kube_controller.go b/pkg/controller/kube_controller.go index 3ff726d..e1f39e6 100644 --- a/pkg/controller/kube_controller.go +++ b/pkg/controller/kube_controller.go @@ -48,6 +48,7 @@ func NewKubeReconciler( log = log.WithField("controller", "channel") httpClient := retryablehttp.NewClient() + httpClient.HTTPClient.Timeout = 10 * time.Second httpClient.RetryMax = 3 httpClient.RetryWaitMin = 1 * time.Second httpClient.RetryWaitMax = 30 * time.Second diff --git a/pkg/metrics/helpers.go b/pkg/metrics/helpers.go index ea3e2ff..5aa637f 100644 --- a/pkg/metrics/helpers.go +++ b/pkg/metrics/helpers.go @@ -31,6 +31,12 @@ func buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL s } } +// buildContainerImageLabels matches the metrics that are keyed by container +// and image but have no container_type label. +func buildContainerImageLabels(namespace, pod, container string) prometheus.Labels { + return prometheus.Labels{"namespace": namespace, "pod": pod, "container": container} +} + func buildPodPartialLabels(namespace, pod string) prometheus.Labels { return prometheus.Labels{ "namespace": namespace, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d37e958..fe449ce 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "strings" "sync" "time" @@ -35,6 +36,11 @@ type Metrics struct { cache k8sclient.Reader + // errorImage remembers which image each container's failure counter is + // for (key "namespace/pod/container"), so the counter of a replaced image + // can be dropped instead of accumulating series. + errorImage map[string]string + // Contains all metrics for the roundtripper roundTripper *RoundTripper @@ -118,8 +124,9 @@ func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, cache k8sclient.R ) return &Metrics{ - log: log.WithField("module", "metrics"), - cache: cache, + log: log.WithField("module", "metrics"), + cache: cache, + errorImage: map[string]string{}, registry: reg, containerImageVersion: containerImageVersion, @@ -168,9 +175,12 @@ func (m *Metrics) RemoveImage(namespace, pod, container, containerType string) { labels := buildContainerPartialLabels(namespace, pod, container, containerType) total += m.containerImageVersion.DeletePartialMatch(labels) - total += m.containerImageDuration.DeletePartialMatch(labels) total += m.containerImageChecked.DeletePartialMatch(labels) - total += m.containerImageErrors.DeletePartialMatch(labels) + // Duration and errors have no container_type label, so a partial match + // that includes it would match nothing. + total += m.containerImageDuration.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) + total += m.containerImageErrors.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) + delete(m.errorImage, namespace+"/"+pod+"/"+container) total += m.containerImageTimestamp.DeletePartialMatch(labels) total += m.containerImageAvailable.DeletePartialMatch(labels) @@ -194,6 +204,11 @@ func (m *Metrics) RemovePod(namespace, pod string) { total += m.containerImageErrors.DeletePartialMatch( buildPodPartialLabels(namespace, pod), ) + for k := range m.errorImage { + if strings.HasPrefix(k, namespace+"/"+pod+"/") { + delete(m.errorImage, k) + } + } total += m.containerImageTimestamp.DeletePartialMatch( buildPodPartialLabels(namespace, pod), ) @@ -213,6 +228,8 @@ func (m *Metrics) RegisterImageDuration(namespace, pod, container, image string, return } + // One series per container: drop the one for a previous image. + m.containerImageDuration.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) m.containerImageDuration.WithLabelValues( namespace, pod, container, image, ).Set(time.Since(startTime).Seconds()) @@ -227,6 +244,14 @@ func (m *Metrics) ReportError(namespace, pod, container, imageURL string) { return } + // A replaced image starts a new failure count rather than a new series + // next to the old one. + key := namespace + "/" + pod + "/" + container + if prev, ok := m.errorImage[key]; ok && prev != imageURL { + m.containerImageErrors.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container)) + } + m.errorImage[key] = imageURL + m.containerImageErrors.WithLabelValues( namespace, pod, container, imageURL, ).Inc() diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index a7847a7..9bc621e 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -278,3 +278,25 @@ func TestPodAnnotationsChangeAfterRegistration(t *testing.T) { testutil.CollectAndCount(metrics.containerImageVersion.MetricVec, MetricNamespace+"_is_latest_version"), ) } + +// A container whose image changes, or that is no longer checked, must not +// leave lookup duration and failure series behind. +func TestImageSeriesDoNotAccumulate(t *testing.T) { + m := New(logrus.NewEntry(logrus.New()), prometheus.NewRegistry(), fakek8s) + ctx := context.Background() + require.NoError(t, fakek8s.DeleteAllOf(ctx, &corev1.Pod{})) + require.NoError(t, fakek8s.Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns"}, + })) + + for _, img := range []string{"a", "b", "c"} { + m.RegisterImageDuration("ns", "p", "c", img, time.Now()) + m.ReportError("ns", "p", "c", img) + } + assert.Equal(t, 1, testutil.CollectAndCount(m.containerImageDuration)) + assert.Equal(t, 1, testutil.CollectAndCount(m.containerImageErrors)) + + m.RemoveImage("ns", "p", "c", "init") + assert.Equal(t, 0, testutil.CollectAndCount(m.containerImageDuration)) + assert.Equal(t, 0, testutil.CollectAndCount(m.containerImageErrors)) +}