diff --git a/pkg/client/client.go b/pkg/client/client.go index 4519204c..0929b972 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/sirupsen/logrus" @@ -21,6 +22,10 @@ import ( "github.com/jetstack/version-checker/pkg/client/util" ) +// lookupTimeout is the maximum time a single image lookup may take. Selfhosted +// registries are queried tag by tag, so very large repos need minutes. +const lookupTimeout = 10 * time.Minute + // Used for testing/mocking purposes type ClientHandler interface { Tags(ctx context.Context, imageURL string) ([]api.ImageTag, error) @@ -125,6 +130,10 @@ func New(ctx context.Context, log *logrus.Entry, opts Options) (*Client, error) // Tags returns the full list of image tags available, for a given image URL. func (c *Client) Tags(ctx context.Context, imageURL string) ([]api.ImageTag, error) { + // Bound total work per lookup: the registry host is pod-controlled and must not be able to pin a reconcile worker. + ctx, cancel := context.WithTimeout(ctx, lookupTimeout) + defer cancel() + client, host, path := c.fromImageURL(imageURL) // The host is pod-controlled and is interpolated into request URLs. if host != "" { diff --git a/pkg/client/docker/docker.go b/pkg/client/docker/docker.go index 581d8cce..eef5fd03 100644 --- a/pkg/client/docker/docker.go +++ b/pkg/client/docker/docker.go @@ -1,13 +1,13 @@ package docker import ( + "bytes" "context" "encoding/json" "errors" "fmt" - "io" "net/http" - "strings" + neturl "net/url" "time" "golang.org/x/time/rate" @@ -98,11 +98,25 @@ func (c *Client) Name() string { return "dockerhub" } +const maxPages = 100 + func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTag, error) { + repo, err := util.SafePath(repo) + if err != nil { + return nil, err + } + image, err = util.SafePath(image) + if err != nil { + return nil, err + } url := fmt.Sprintf(lookupURL, repo, image) var tags []api.ImageTag - for url != "" { + for page := 0; url != ""; page++ { + if page >= maxPages { + c.log.Warnf("%s/%s: more than %d tag pages, only the newest %d tags are checked", repo, image, maxPages, len(tags)) + break + } response, err := c.doRequest(ctx, url) if err != nil { return nil, err @@ -157,6 +171,11 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa } url = response.Next + if url != "" { + if u, err := neturl.Parse(url); err != nil || u.Host != "registry.hub.docker.com" { + return nil, fmt.Errorf("refusing to follow next page URL %q", url) + } + } } return tags, nil @@ -181,25 +200,21 @@ func (c *Client) doRequest(ctx context.Context, url string) (*TagResponse, error } defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) + body, err := util.ReadAll(resp.Body) if err != nil { return nil, err } response := new(TagResponse) if err := json.Unmarshal(body, response); err != nil { - return nil, fmt.Errorf("unexpected image tags response: %s", body) + return nil, fmt.Errorf("unexpected image tags response: %s", util.Truncate(body)) } return response, nil } func basicAuthSetup(ctx context.Context, client *http.Client, opts Options) (string, error) { - upReader := strings.NewReader( - fmt.Sprintf(`{"username": "%s", "password": "%s"}`, - opts.Username, opts.Password, - ), - ) + upReader := bytes.NewReader(util.LoginBody(opts.Username, opts.Password)) req, err := http.NewRequest(http.MethodPost, loginURL, upReader) if err != nil { @@ -216,13 +231,13 @@ func basicAuthSetup(ctx context.Context, client *http.Client, opts Options) (str } defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) + body, err := util.ReadAll(resp.Body) if err != nil { return "", err } if resp.StatusCode != http.StatusOK { - return "", errors.New(string(body)) + return "", errors.New(util.Truncate(body)) } response := new(AuthResponse) diff --git a/pkg/client/gcr/gcr.go b/pkg/client/gcr/gcr.go index 67eefaab..b1121231 100644 --- a/pkg/client/gcr/gcr.go +++ b/pkg/client/gcr/gcr.go @@ -4,12 +4,12 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "strconv" "time" "github.com/jetstack/version-checker/pkg/api" + "github.com/jetstack/version-checker/pkg/client/util" ) const ( @@ -54,7 +54,10 @@ func (c *Client) Name() string { } func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.ImageTag, error) { - image = c.constructImageName(repo, image) + image, err := util.SafePath(c.constructImageName(repo, image)) + if err != nil { + return nil, err + } url := fmt.Sprintf(lookupURL, host, image) req, err := c.buildRequest(ctx, url) @@ -68,7 +71,7 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag } defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) + body, err := util.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/pkg/client/ghcr/ghcr.go b/pkg/client/ghcr/ghcr.go index 80a2e5e2..7c931aa6 100644 --- a/pkg/client/ghcr/ghcr.go +++ b/pkg/client/ghcr/ghcr.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "strings" + "sync" "github.com/jetstack/version-checker/pkg/api" @@ -24,7 +25,7 @@ type Options struct { type Client struct { client *github.Client - ownerTypes map[string]string + ownerTypes sync.Map // owner -> lowercased owner type; read by concurrent workers opts Options } @@ -47,9 +48,8 @@ func New(opts Options) *Client { } return &Client{ - client: client, - opts: opts, - ownerTypes: map[string]string{}, + client: client, + opts: opts, } } @@ -111,17 +111,17 @@ func (c *Client) buildPackageListOptions() *github.PackageListOptions { func (c *Client) extractImageTags(versions []*github.PackageVersion) []api.ImageTag { tags := map[string]api.ImageTag{} for _, ver := range versions { - if meta, ok := ver.GetMetadata(); ok { + if meta, ok := ver.GetMetadata(); ok && meta != nil && meta.Container != nil { var sha string - if strings.HasPrefix(*ver.Name, "sha") { - sha = *ver.Name + if strings.HasPrefix(ver.GetName(), "sha") { + sha = ver.GetName() } base := api.ImageTag{ - Tag: *ver.Name, + Tag: ver.GetName(), SHA: sha, - Timestamp: ver.CreatedAt.Time, + Timestamp: ver.GetCreatedAt().Time, } for _, tag := range meta.Container.Tags { @@ -149,8 +149,8 @@ func (c *Client) extractImageTags(versions []*github.PackageVersion) []api.Image } func (c *Client) ownerType(ctx context.Context, owner string) (string, error) { - if ownerType, ok := c.ownerTypes[owner]; ok { - return ownerType, nil + if ownerType, ok := c.ownerTypes.Load(owner); ok { + return ownerType.(string), nil } user, _, err := c.client.Users.Get(ctx, owner) if err != nil { @@ -158,7 +158,7 @@ func (c *Client) ownerType(ctx context.Context, owner string) (string, error) { } ownerType := strings.ToLower(user.GetType()) - c.ownerTypes[owner] = ownerType + c.ownerTypes.Store(owner, ownerType) return ownerType, nil } diff --git a/pkg/client/ghcr/nil_test.go b/pkg/client/ghcr/nil_test.go new file mode 100644 index 00000000..d4eaf125 --- /dev/null +++ b/pkg/client/ghcr/nil_test.go @@ -0,0 +1,27 @@ +package ghcr + +import ( + "encoding/json" + "testing" + + "github.com/google/go-github/v70/github" + "github.com/stretchr/testify/assert" +) + +// The GitHub API can omit created_at, metadata or metadata.container. +func TestExtractImageTagsToleratesMissingFields(t *testing.T) { + c := &Client{} + versions := []*github.PackageVersion{ + {Name: github.Ptr("no-metadata")}, + {Name: github.Ptr("null-metadata"), Metadata: json.RawMessage(`null`)}, + {Name: github.Ptr("no-container"), Metadata: json.RawMessage(`{}`)}, + {Name: github.Ptr("no-created-at"), Metadata: json.RawMessage(`{"container":{"tags":["v1"]}}`)}, + } + var tags []string + assert.NotPanics(t, func() { + for _, tag := range c.extractImageTags(versions) { + tags = append(tags, tag.Tag) + } + }) + assert.Equal(t, []string{"v1"}, tags) +} diff --git a/pkg/client/oci/oci.go b/pkg/client/oci/oci.go index 3ab51fcb..a2071db0 100644 --- a/pkg/client/oci/oci.go +++ b/pkg/client/oci/oci.go @@ -25,6 +25,9 @@ type Options struct { Auth *authn.AuthConfig } +// maxTags bounds the per-tag manifest requests made for one lookup. +const maxTags = 5000 + func (o *Options) Authorization() (*authn.AuthConfig, error) { if o.Auth != nil { return o.Auth, nil @@ -81,6 +84,10 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag if err != nil { return nil, fmt.Errorf("listing tags: %w", err) } + if len(bareTags) > maxTags { + c.log.Warnf("%s: registry returned %d tags, only the first %d are checked", reg.Repo(repo, image), len(bareTags), maxTags) + bareTags = bareTags[:maxTags] + } c.log.Infof("Collected %v tags..", len(bareTags)) return c.Manifests(ctx, reg.Repo(repo, image), bareTags) } diff --git a/pkg/client/quay/pager.go b/pkg/client/quay/pager.go index c99a81f7..2b260d5d 100644 --- a/pkg/client/quay/pager.go +++ b/pkg/client/quay/pager.go @@ -2,6 +2,7 @@ package quay import ( "context" + "errors" "fmt" "sync" @@ -30,6 +31,10 @@ func (c *Client) newPager(repo, image string) *pager { } func (p *pager) fetchTags(ctx context.Context) error { + // Always wait for in-flight manifest fetches, including on error paths, so + // no goroutine outlives the call or races with the caller reading p.tags. + defer p.wg.Wait() + var ( page = 1 hasAdditional = true @@ -49,6 +54,13 @@ func (p *pager) fetchTags(ctx context.Context) error { p.wg.Wait() + // Individual manifest failures skip a tag; only fail when nothing could be fetched. + p.mu.Lock() + defer p.mu.Unlock() + if len(p.tags) == 0 && len(p.errs) > 0 { + return errors.Join(p.errs...) + } + return nil } diff --git a/pkg/client/quay/quay.go b/pkg/client/quay/quay.go index bb745105..bfb5f18d 100644 --- a/pkg/client/quay/quay.go +++ b/pkg/client/quay/quay.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" + neturl "net/url" "time" "github.com/hashicorp/go-retryablehttp" @@ -54,6 +56,14 @@ func (c *Client) Name() string { // Fetch the image tags from an upstream repository and image. func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTag, error) { + repo, err := util.SafePath(repo) + if err != nil { + return nil, err + } + image, err = util.SafePath(image) + if err != nil { + return nil, err + } p := c.newPager(repo, image) if err := p.fetchTags(ctx); err != nil { @@ -80,7 +90,7 @@ func (c *Client) fetchImageManifest(ctx context.Context, repo, image string, tag // If a multi-arch image, call manifest endpoint if tag.IsManifestList { - url := fmt.Sprintf(manifestURL, repo, image, tag.ManifestDigest) + url := fmt.Sprintf(manifestURL, repo, image, neturl.PathEscape(tag.ManifestDigest)) err := c.callManifests(ctx, timestamp, iTag, url) if err != nil { return nil, err @@ -147,7 +157,7 @@ func (c *Client) makeRequest(ctx context.Context, url string, obj interface{}) e } defer func() { _ = resp.Body.Close() }() - if err := json.NewDecoder(resp.Body).Decode(obj); err != nil { + if err := json.NewDecoder(io.LimitReader(resp.Body, util.MaxResponseBytes)).Decode(obj); err != nil { return err } diff --git a/pkg/client/selfhosted/selfhosted.go b/pkg/client/selfhosted/selfhosted.go index ef475e16..6761523e 100644 --- a/pkg/client/selfhosted/selfhosted.go +++ b/pkg/client/selfhosted/selfhosted.go @@ -1,13 +1,13 @@ package selfhosted import ( + "bytes" "context" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" - "io" "net/http" "os" "regexp" @@ -103,6 +103,12 @@ func configureHost(ctx context.Context, client *Client, opts *Options) error { } func configureAuth(ctx context.Context, client *Client, opts *Options) error { + if len(opts.Username) == 0 && len(opts.Password) == 0 && len(opts.Bearer) == 0 { + return nil + } + if client.httpScheme == "http" { + client.log.Warnf("registry %q uses plain http: credentials are sent in cleartext", opts.Host) + } if len(opts.Username) == 0 && len(opts.Password) == 0 { return nil } @@ -164,8 +170,14 @@ func (c *Client) Name() string { // Tags will fetch the image tags from a given image URL. It must first query // the tags that are available, then query the 2.1 and 2.2 API endpoints to // gather the image digest and created time. +// maxTags bounds the per-tag manifest requests made for one lookup. +const maxTags = 5000 + func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.ImageTag, error) { - path := util.JoinRepoImage(repo, image) + path, err := util.SafePath(util.JoinRepoImage(repo, image)) + if err != nil { + return nil, err + } tagURL := fmt.Sprintf(tagsPath, host, path) var tagResponse TagResponse @@ -173,6 +185,11 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag return nil, err } + if len(tagResponse.Tags) > maxTags { + c.log.Warnf("%s: registry returned %d tags, only the first %d are checked", path, len(tagResponse.Tags), maxTags) + tagResponse.Tags = tagResponse.Tags[:maxTags] + } + tags := map[string]api.ImageTag{} for _, tag := range tagResponse.Tags { manifestURL := fmt.Sprintf(manifestPath, host, path, tag) @@ -259,29 +276,24 @@ func (c *Client) doRequest(ctx context.Context, url, header string, obj interfac } defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) + body, err := util.ReadAll(resp.Body) if err != nil { return nil, err } - defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return nil, selfhostederrors.NewHTTPError(resp.StatusCode, body) + return nil, selfhostederrors.NewHTTPError(resp.StatusCode, body[:min(len(body), 256)]) } if err := json.Unmarshal(body, obj); err != nil { - return nil, fmt.Errorf("unexpected %s response: %s - %w", url, body, err) + return nil, fmt.Errorf("unexpected %s response: %s - %w", url, util.Truncate(body), err) } return resp.Header, nil } func (c *Client) setupBasicAuth(ctx context.Context, url, tokenPath string) (string, error) { - upReader := strings.NewReader( - fmt.Sprintf(`{"username": "%s", "password": "%s"}`, - c.Username, c.Password, - ), - ) + upReader := bytes.NewReader(util.LoginBody(c.Username, c.Password)) tokenURL := url + tokenPath @@ -300,13 +312,13 @@ func (c *Client) setupBasicAuth(ctx context.Context, url, tokenPath string) (str } defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) + body, err := util.ReadAll(resp.Body) if err != nil { return "", err } if resp.StatusCode != http.StatusOK { - return "", selfhostederrors.NewHTTPError(resp.StatusCode, body) + return "", selfhostederrors.NewHTTPError(resp.StatusCode, body[:min(len(body), 256)]) } response := new(AuthResponse) diff --git a/pkg/client/util/creds.go b/pkg/client/util/creds.go new file mode 100644 index 00000000..5cb9e0b9 --- /dev/null +++ b/pkg/client/util/creds.go @@ -0,0 +1,9 @@ +package util + +import "encoding/json" + +// LoginBody returns a correctly escaped JSON login body. +func LoginBody(user, pass string) []byte { + b, _ := json.Marshal(map[string]string{"username": user, "password": pass}) + return b +} diff --git a/pkg/client/util/safe.go b/pkg/client/util/safe.go new file mode 100644 index 00000000..5ff54ab1 --- /dev/null +++ b/pkg/client/util/safe.go @@ -0,0 +1,48 @@ +package util + +import ( + "fmt" + "io" + "net/url" + "strings" +) + +// MaxResponseBytes caps how much of a registry response is read into memory. +const MaxResponseBytes = 16 << 20 + +// ReadAll reads r, failing if it holds more than MaxResponseBytes. +func ReadAll(r io.Reader) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, MaxResponseBytes+1)) + if err != nil { + return nil, err + } + if len(b) > MaxResponseBytes { + return nil, fmt.Errorf("response exceeds %d bytes", MaxResponseBytes) + } + return b, nil +} + +// Truncate shortens b for logs and error strings, so remote bodies are not reflected whole. +func Truncate(b []byte) string { + const max = 256 + if len(b) > max { + return fmt.Sprintf("%q...[truncated %d bytes]", b[:max], len(b)-max) + } + return fmt.Sprintf("%q", b) +} + +// SafePath escapes each segment of a "/"-separated repository path so it +// cannot alter the request path or query. Empty input returns "". +func SafePath(p string) (string, error) { + if p == "" { + return "", nil + } + segs := strings.Split(p, "/") + for i, s := range segs { + if s == "" || s == "." || s == ".." { + return "", fmt.Errorf("invalid path %q", p) + } + segs[i] = url.PathEscape(s) + } + return strings.Join(segs, "/"), nil +} diff --git a/pkg/client/util/safe_test.go b/pkg/client/util/safe_test.go new file mode 100644 index 00000000..3901e79d --- /dev/null +++ b/pkg/client/util/safe_test.go @@ -0,0 +1,14 @@ +package util + +import "testing" + +func TestSafePath(t *testing.T) { + for _, bad := range []string{"a/../b", "a//b", "../x", "."} { + if _, err := SafePath(bad); err == nil { + t.Errorf("SafePath(%q) should fail", bad) + } + } + if got, _ := SafePath("a/b?c"); got != "a/b%3Fc" { + t.Errorf("got %q", got) + } +}