Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions cmd/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ func NewCommand(ctx context.Context) *cobra.Command {
Long: helpOutput,
RunE: func(_ *cobra.Command, _ []string) error {
opts.complete()
if err := opts.validateIntervals(); err != nil {
return err
}

logLevel, err := logrus.ParseLevel(opts.LogLevel)
if err != nil {
Expand Down Expand Up @@ -94,8 +97,8 @@ func NewCommand(ctx context.Context) *cobra.Command {

// Readiness probe
if err := mgr.AddMetricsServerExtraHandler("/readyz",
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if mgr.GetCache().WaitForCacheSync(context.Background()) {
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if mgr.GetCache().WaitForCacheSync(r.Context()) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ready"))
} else {
Expand Down
9 changes: 9 additions & 0 deletions cmd/app/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,15 @@ func (o *Options) assignSelfhosted(envs []string) {
}
}

// validateIntervals rejects durations that would panic time.NewTicker or
// disable the cache.
func (o *Options) validateIntervals() error {
if o.CacheTimeout <= 0 || o.KubeInterval <= 0 {
return fmt.Errorf("--image-cache-timeout and --kube-interval must be greater than zero")
}
return nil
}

func validSelfHostedOpts(opts *Options) bool {
// opts set using env vars
if opts.Client.Selfhosted != nil {
Expand Down
7 changes: 7 additions & 0 deletions cmd/app/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package app
import (
"os"
"testing"
"time"

"github.com/stretchr/testify/assert"

Expand Down Expand Up @@ -359,3 +360,9 @@ func TestAssignSelfhosted(t *testing.T) {
})
}
}

func TestValidateIntervals(t *testing.T) {
assert.NoError(t, (&Options{CacheTimeout: time.Minute, KubeInterval: time.Minute}).validateIntervals())
assert.Error(t, (&Options{CacheTimeout: 0, KubeInterval: time.Minute}).validateIntervals())
assert.Error(t, (&Options{CacheTimeout: time.Minute, KubeInterval: -1}).validateIntervals())
}
1 change: 1 addition & 0 deletions pkg/controller/kube_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func NewKubeReconciler(
log = log.WithField("controller", "channel")

httpClient := retryablehttp.NewClient()
httpClient.HTTPClient.Timeout = 10 * time.Second
httpClient.RetryMax = 3
httpClient.RetryWaitMin = 1 * time.Second
httpClient.RetryWaitMax = 30 * time.Second
Expand Down
6 changes: 6 additions & 0 deletions pkg/metrics/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ func buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL s
}
}

// buildContainerImageLabels matches the metrics that are keyed by container
// and image but have no container_type label.
func buildContainerImageLabels(namespace, pod, container string) prometheus.Labels {
return prometheus.Labels{"namespace": namespace, "pod": pod, "container": container}
}

func buildPodPartialLabels(namespace, pod string) prometheus.Labels {
return prometheus.Labels{
"namespace": namespace,
Expand Down
33 changes: 29 additions & 4 deletions pkg/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package metrics

import (
"context"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -35,6 +36,11 @@ type Metrics struct {

cache k8sclient.Reader

// errorImage remembers which image each container's failure counter is
// for (key "namespace/pod/container"), so the counter of a replaced image
// can be dropped instead of accumulating series.
errorImage map[string]string

// Contains all metrics for the roundtripper
roundTripper *RoundTripper

Expand Down Expand Up @@ -118,8 +124,9 @@ func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, cache k8sclient.R
)

return &Metrics{
log: log.WithField("module", "metrics"),
cache: cache,
log: log.WithField("module", "metrics"),
cache: cache,
errorImage: map[string]string{},

registry: reg,
containerImageVersion: containerImageVersion,
Expand Down Expand Up @@ -168,9 +175,12 @@ func (m *Metrics) RemoveImage(namespace, pod, container, containerType string) {
labels := buildContainerPartialLabels(namespace, pod, container, containerType)

total += m.containerImageVersion.DeletePartialMatch(labels)
total += m.containerImageDuration.DeletePartialMatch(labels)
total += m.containerImageChecked.DeletePartialMatch(labels)
total += m.containerImageErrors.DeletePartialMatch(labels)
// Duration and errors have no container_type label, so a partial match
// that includes it would match nothing.
total += m.containerImageDuration.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container))
total += m.containerImageErrors.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container))
delete(m.errorImage, namespace+"/"+pod+"/"+container)
total += m.containerImageTimestamp.DeletePartialMatch(labels)
total += m.containerImageAvailable.DeletePartialMatch(labels)

Expand All @@ -194,6 +204,11 @@ func (m *Metrics) RemovePod(namespace, pod string) {
total += m.containerImageErrors.DeletePartialMatch(
buildPodPartialLabels(namespace, pod),
)
for k := range m.errorImage {
if strings.HasPrefix(k, namespace+"/"+pod+"/") {
delete(m.errorImage, k)
}
}
total += m.containerImageTimestamp.DeletePartialMatch(
buildPodPartialLabels(namespace, pod),
)
Expand All @@ -213,6 +228,8 @@ func (m *Metrics) RegisterImageDuration(namespace, pod, container, image string,
return
}

// One series per container: drop the one for a previous image.
m.containerImageDuration.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container))
m.containerImageDuration.WithLabelValues(
namespace, pod, container, image,
).Set(time.Since(startTime).Seconds())
Expand All @@ -227,6 +244,14 @@ func (m *Metrics) ReportError(namespace, pod, container, imageURL string) {
return
}

// A replaced image starts a new failure count rather than a new series
// next to the old one.
key := namespace + "/" + pod + "/" + container
if prev, ok := m.errorImage[key]; ok && prev != imageURL {
m.containerImageErrors.DeletePartialMatch(buildContainerImageLabels(namespace, pod, container))
}
m.errorImage[key] = imageURL

m.containerImageErrors.WithLabelValues(
namespace, pod, container, imageURL,
).Inc()
Expand Down
22 changes: 22 additions & 0 deletions pkg/metrics/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,3 +278,25 @@ func TestPodAnnotationsChangeAfterRegistration(t *testing.T) {
testutil.CollectAndCount(metrics.containerImageVersion.MetricVec, MetricNamespace+"_is_latest_version"),
)
}

// A container whose image changes, or that is no longer checked, must not
// leave lookup duration and failure series behind.
func TestImageSeriesDoNotAccumulate(t *testing.T) {
m := New(logrus.NewEntry(logrus.New()), prometheus.NewRegistry(), fakek8s)
ctx := context.Background()
require.NoError(t, fakek8s.DeleteAllOf(ctx, &corev1.Pod{}))
require.NoError(t, fakek8s.Create(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns"},
}))

for _, img := range []string{"a", "b", "c"} {
m.RegisterImageDuration("ns", "p", "c", img, time.Now())
m.ReportError("ns", "p", "c", img)
}
assert.Equal(t, 1, testutil.CollectAndCount(m.containerImageDuration))
assert.Equal(t, 1, testutil.CollectAndCount(m.containerImageErrors))

m.RemoveImage("ns", "p", "c", "init")
assert.Equal(t, 0, testutil.CollectAndCount(m.containerImageDuration))
assert.Equal(t, 0, testutil.CollectAndCount(m.containerImageErrors))
}