Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions pkg/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"strings"
"time"

"github.com/sirupsen/logrus"

Expand All @@ -21,6 +22,10 @@ import (
"github.com/jetstack/version-checker/pkg/client/util"
)

// lookupTimeout is the maximum time a single image lookup may take. Selfhosted
// registries are queried tag by tag, so very large repos need minutes.
const lookupTimeout = 10 * time.Minute

// Used for testing/mocking purposes
type ClientHandler interface {
Tags(ctx context.Context, imageURL string) ([]api.ImageTag, error)
Expand Down Expand Up @@ -125,6 +130,10 @@ func New(ctx context.Context, log *logrus.Entry, opts Options) (*Client, error)

// Tags returns the full list of image tags available, for a given image URL.
func (c *Client) Tags(ctx context.Context, imageURL string) ([]api.ImageTag, error) {
// Bound total work per lookup: the registry host is pod-controlled and must not be able to pin a reconcile worker.
ctx, cancel := context.WithTimeout(ctx, lookupTimeout)
defer cancel()

client, host, path := c.fromImageURL(imageURL)
// The host is pod-controlled and is interpolated into request URLs.
if host != "" {
Expand Down
39 changes: 27 additions & 12 deletions pkg/client/docker/docker.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package docker

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
neturl "net/url"
"time"

"golang.org/x/time/rate"
Expand Down Expand Up @@ -98,11 +98,25 @@ func (c *Client) Name() string {
return "dockerhub"
}

const maxPages = 100

func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTag, error) {
repo, err := util.SafePath(repo)
if err != nil {
return nil, err
}
image, err = util.SafePath(image)
if err != nil {
return nil, err
}
url := fmt.Sprintf(lookupURL, repo, image)

var tags []api.ImageTag
for url != "" {
for page := 0; url != ""; page++ {
if page >= maxPages {
c.log.Warnf("%s/%s: more than %d tag pages, only the newest %d tags are checked", repo, image, maxPages, len(tags))
break
}
response, err := c.doRequest(ctx, url)
if err != nil {
return nil, err
Expand Down Expand Up @@ -157,6 +171,11 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa
}

url = response.Next
if url != "" {
if u, err := neturl.Parse(url); err != nil || u.Host != "registry.hub.docker.com" {
return nil, fmt.Errorf("refusing to follow next page URL %q", url)
}
}
}

return tags, nil
Expand All @@ -181,25 +200,21 @@ func (c *Client) doRequest(ctx context.Context, url string) (*TagResponse, error
}
defer func() { _ = resp.Body.Close() }()

body, err := io.ReadAll(resp.Body)
body, err := util.ReadAll(resp.Body)
if err != nil {
return nil, err
}

response := new(TagResponse)
if err := json.Unmarshal(body, response); err != nil {
return nil, fmt.Errorf("unexpected image tags response: %s", body)
return nil, fmt.Errorf("unexpected image tags response: %s", util.Truncate(body))
}

return response, nil
}

func basicAuthSetup(ctx context.Context, client *http.Client, opts Options) (string, error) {
upReader := strings.NewReader(
fmt.Sprintf(`{"username": "%s", "password": "%s"}`,
opts.Username, opts.Password,
),
)
upReader := bytes.NewReader(util.LoginBody(opts.Username, opts.Password))

req, err := http.NewRequest(http.MethodPost, loginURL, upReader)
if err != nil {
Expand All @@ -216,13 +231,13 @@ func basicAuthSetup(ctx context.Context, client *http.Client, opts Options) (str
}
defer func() { _ = resp.Body.Close() }()

body, err := io.ReadAll(resp.Body)
body, err := util.ReadAll(resp.Body)
if err != nil {
return "", err
}

if resp.StatusCode != http.StatusOK {
return "", errors.New(string(body))
return "", errors.New(util.Truncate(body))
}

response := new(AuthResponse)
Expand Down
9 changes: 6 additions & 3 deletions pkg/client/gcr/gcr.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"time"

"github.com/jetstack/version-checker/pkg/api"
"github.com/jetstack/version-checker/pkg/client/util"
)

const (
Expand Down Expand Up @@ -54,7 +54,10 @@ func (c *Client) Name() string {
}

func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.ImageTag, error) {
image = c.constructImageName(repo, image)
image, err := util.SafePath(c.constructImageName(repo, image))
if err != nil {
return nil, err
}
url := fmt.Sprintf(lookupURL, host, image)

req, err := c.buildRequest(ctx, url)
Expand All @@ -68,7 +71,7 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag
}
defer func() { _ = resp.Body.Close() }()

body, err := io.ReadAll(resp.Body)
body, err := util.ReadAll(resp.Body)
if err != nil {
return nil, err
}
Expand Down
24 changes: 12 additions & 12 deletions pkg/client/ghcr/ghcr.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net/http"
"net/url"
"strings"
"sync"

"github.com/jetstack/version-checker/pkg/api"

Expand All @@ -24,7 +25,7 @@ type Options struct {

type Client struct {
client *github.Client
ownerTypes map[string]string
ownerTypes sync.Map // owner -> lowercased owner type; read by concurrent workers
opts Options
}

Expand All @@ -47,9 +48,8 @@ func New(opts Options) *Client {
}

return &Client{
client: client,
opts: opts,
ownerTypes: map[string]string{},
client: client,
opts: opts,
}
}

Expand Down Expand Up @@ -111,17 +111,17 @@ func (c *Client) buildPackageListOptions() *github.PackageListOptions {
func (c *Client) extractImageTags(versions []*github.PackageVersion) []api.ImageTag {
tags := map[string]api.ImageTag{}
for _, ver := range versions {
if meta, ok := ver.GetMetadata(); ok {
if meta, ok := ver.GetMetadata(); ok && meta != nil && meta.Container != nil {

var sha string
if strings.HasPrefix(*ver.Name, "sha") {
sha = *ver.Name
if strings.HasPrefix(ver.GetName(), "sha") {
sha = ver.GetName()
}

base := api.ImageTag{
Tag: *ver.Name,
Tag: ver.GetName(),
SHA: sha,
Timestamp: ver.CreatedAt.Time,
Timestamp: ver.GetCreatedAt().Time,
}

for _, tag := range meta.Container.Tags {
Expand Down Expand Up @@ -149,16 +149,16 @@ func (c *Client) extractImageTags(versions []*github.PackageVersion) []api.Image
}

func (c *Client) ownerType(ctx context.Context, owner string) (string, error) {
if ownerType, ok := c.ownerTypes[owner]; ok {
return ownerType, nil
if ownerType, ok := c.ownerTypes.Load(owner); ok {
return ownerType.(string), nil
}
user, _, err := c.client.Users.Get(ctx, owner)
if err != nil {
return "", fmt.Errorf("fetching user: %w", err)
}
ownerType := strings.ToLower(user.GetType())

c.ownerTypes[owner] = ownerType
c.ownerTypes.Store(owner, ownerType)

return ownerType, nil
}
27 changes: 27 additions & 0 deletions pkg/client/ghcr/nil_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package ghcr

import (
"encoding/json"
"testing"

"github.com/google/go-github/v70/github"
"github.com/stretchr/testify/assert"
)

// The GitHub API can omit created_at, metadata or metadata.container.
func TestExtractImageTagsToleratesMissingFields(t *testing.T) {
c := &Client{}
versions := []*github.PackageVersion{
{Name: github.Ptr("no-metadata")},
{Name: github.Ptr("null-metadata"), Metadata: json.RawMessage(`null`)},
{Name: github.Ptr("no-container"), Metadata: json.RawMessage(`{}`)},
{Name: github.Ptr("no-created-at"), Metadata: json.RawMessage(`{"container":{"tags":["v1"]}}`)},
}
var tags []string
assert.NotPanics(t, func() {
for _, tag := range c.extractImageTags(versions) {
tags = append(tags, tag.Tag)
}
})
assert.Equal(t, []string{"v1"}, tags)
}
7 changes: 7 additions & 0 deletions pkg/client/oci/oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ type Options struct {
Auth *authn.AuthConfig
}

// maxTags bounds the per-tag manifest requests made for one lookup.
const maxTags = 5000

func (o *Options) Authorization() (*authn.AuthConfig, error) {
if o.Auth != nil {
return o.Auth, nil
Expand Down Expand Up @@ -81,6 +84,10 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag
if err != nil {
return nil, fmt.Errorf("listing tags: %w", err)
}
if len(bareTags) > maxTags {
c.log.Warnf("%s: registry returned %d tags, only the first %d are checked", reg.Repo(repo, image), len(bareTags), maxTags)
bareTags = bareTags[:maxTags]
}
c.log.Infof("Collected %v tags..", len(bareTags))
return c.Manifests(ctx, reg.Repo(repo, image), bareTags)
}
Expand Down
12 changes: 12 additions & 0 deletions pkg/client/quay/pager.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package quay

import (
"context"
"errors"
"fmt"
"sync"

Expand Down Expand Up @@ -30,6 +31,10 @@ func (c *Client) newPager(repo, image string) *pager {
}

func (p *pager) fetchTags(ctx context.Context) error {
// Always wait for in-flight manifest fetches, including on error paths, so
// no goroutine outlives the call or races with the caller reading p.tags.
defer p.wg.Wait()

var (
page = 1
hasAdditional = true
Expand All @@ -49,6 +54,13 @@ func (p *pager) fetchTags(ctx context.Context) error {

p.wg.Wait()

// Individual manifest failures skip a tag; only fail when nothing could be fetched.
p.mu.Lock()
defer p.mu.Unlock()
if len(p.tags) == 0 && len(p.errs) > 0 {
return errors.Join(p.errs...)
}

return nil
}

Expand Down
14 changes: 12 additions & 2 deletions pkg/client/quay/quay.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
neturl "net/url"
"time"

"github.com/hashicorp/go-retryablehttp"
Expand Down Expand Up @@ -54,6 +56,14 @@ func (c *Client) Name() string {

// Fetch the image tags from an upstream repository and image.
func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTag, error) {
repo, err := util.SafePath(repo)
if err != nil {
return nil, err
}
image, err = util.SafePath(image)
if err != nil {
return nil, err
}
p := c.newPager(repo, image)

if err := p.fetchTags(ctx); err != nil {
Expand All @@ -80,7 +90,7 @@ func (c *Client) fetchImageManifest(ctx context.Context, repo, image string, tag

// If a multi-arch image, call manifest endpoint
if tag.IsManifestList {
url := fmt.Sprintf(manifestURL, repo, image, tag.ManifestDigest)
url := fmt.Sprintf(manifestURL, repo, image, neturl.PathEscape(tag.ManifestDigest))
err := c.callManifests(ctx, timestamp, iTag, url)
if err != nil {
return nil, err
Expand Down Expand Up @@ -147,7 +157,7 @@ func (c *Client) makeRequest(ctx context.Context, url string, obj interface{}) e
}
defer func() { _ = resp.Body.Close() }()

if err := json.NewDecoder(resp.Body).Decode(obj); err != nil {
if err := json.NewDecoder(io.LimitReader(resp.Body, util.MaxResponseBytes)).Decode(obj); err != nil {
return err
}

Expand Down
Loading