Allow configuring timeout for external sources

README.md

@@ -277,9 +277,12 @@ feature is currently disabled by default. To enable this feature add the followi
 ```yaml
 external-sources:
   enable: true
+  abort-after: 10m
   maven:
     search-upstream-by-sha1: true
     base-url: https://repo1.maven.org/maven2
+    abort-after: 5m #override the global config
+
 ```
 
 You can also configure the base-url if you're using another registry as your maven endpoint.

cmd/grype/cli/options/datasources.go

@@ -1,26 +1,31 @@
 package options
 
 import (
+	"github.com/anchore/clio"
+	"time"
 	"github.com/anchore/clio"
 	"github.com/anchore/grype/grype/matcher/java"
 )
 
 const (
 	defaultMavenBaseURL = "https://search.maven.org/solrsearch/select"
+	defaultAbortAfter   = 10 * time.Minute
 )
 
 type externalSources struct {
-	Enable bool  `yaml:"enable" json:"enable" mapstructure:"enable"`
-	Maven  maven `yaml:"maven" json:"maven" mapstructure:"maven"`
+	Enable     bool           `yaml:"enable" json:"enable" mapstructure:"enable"`
+	AbortAfter *time.Duration `yaml:"abort-after" json:"abortAfter" mapstructure:"abort-after"`
+	Maven      maven          `yaml:"maven" json:"maven" mapstructure:"maven"`
 }
 
 var _ interface {
 	clio.FieldDescriber
 } = (*externalSources)(nil)
 
 type maven struct {
-	SearchUpstreamBySha1 bool   `yaml:"search-upstream" json:"searchUpstreamBySha1" mapstructure:"search-maven-upstream"`
-	BaseURL              string `yaml:"base-url" json:"baseUrl" mapstructure:"base-url"`
+	SearchUpstreamBySha1 bool           `yaml:"search-upstream" json:"searchUpstreamBySha1" mapstructure:"search-maven-upstream"`
+	BaseURL              string         `yaml:"base-url" json:"baseUrl" mapstructure:"base-url"`
+	AbortAfter           *time.Duration `yaml:"abort-after" json:"abortAfter" mapstructure:"abort-after"`
 }
 
 func defaultExternalSources() externalSources {
@@ -32,16 +37,33 @@
 	}
 }
 
-func (cfg externalSources) ToJavaMatcherConfig() java.ExternalSearchConfig {
+func (cfg *externalSources) PostLoad() error {
 	// always respect if global config is disabled
-	smu := cfg.Maven.SearchUpstreamBySha1
 	if !cfg.Enable {
-		smu = cfg.Enable
+		cfg.Maven.SearchUpstreamBySha1 = false
 	}
+
+	cfg.Maven.AbortAfter = multiLevelOption[time.Duration](defaultAbortAfter, cfg.AbortAfter, cfg.Maven.AbortAfter)
+
+	return nil
+}
+
+func (cfg externalSources) ToJavaMatcherConfig() java.ExternalSearchConfig {
 	return java.ExternalSearchConfig{
-		SearchMavenUpstream: smu,
+		SearchMavenUpstream: cfg.Maven.SearchUpstreamBySha1,
 		MavenBaseURL:        cfg.Maven.BaseURL,
+		AbortAfter:          *cfg.Maven.AbortAfter,
+	}
+}
+
+func multiLevelOption[T any](defaultValue T, option ...*T) *T {
+	result := defaultValue
+	for _, opt := range option {
+		if opt != nil {
+			result = *opt
+		}
 	}
+	return &result
 }
 
 func (cfg *externalSources) DescribeFields(descriptions clio.FieldDescriptionSet) {

grype/matcher/java/matcher.go

@@ -1,8 +1,10 @@
 package java
 
 import (
+	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/anchore/grype/grype/distro"
 	"github.com/anchore/grype/grype/match"
@@ -25,6 +27,7 @@ type Matcher struct {
 type ExternalSearchConfig struct {
 	SearchMavenUpstream bool
 	MavenBaseURL        string
+	AbortAfter          time.Duration
 }
 
 type MatcherConfig struct {
@@ -53,7 +56,15 @@ func (m *Matcher) Type() match.MatcherType {
 func (m *Matcher) Match(store vulnerability.Provider, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
 	var matches []match.Match
 	if m.cfg.SearchMavenUpstream {
-		upstreamMatches, err := m.matchUpstreamMavenPackages(store, d, p)
+		timeout := m.cfg.AbortAfter
+		ctx := context.Background()
+		if timeout > 0 {
+			var cancel context.CancelFunc
+			ctx, cancel = context.WithTimeout(ctx, m.cfg.AbortAfter)
+			defer cancel()
+		}
+
+		upstreamMatches, err := m.matchUpstreamMavenPackages(ctx, store, d, p)
 		if err != nil {
 			log.Debugf("failed to match against upstream data for %s: %v", p.Name, err)
 		} else {
@@ -73,13 +84,13 @@ func (m *Matcher) Match(store vulnerability.Provider, d *distro.Distro, p pkg.Pa
 	return matches, nil
 }
 
-func (m *Matcher) matchUpstreamMavenPackages(store vulnerability.Provider, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
+func (m *Matcher) matchUpstreamMavenPackages(ctx context.Context, store vulnerability.Provider, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
 	var matches []match.Match
 
 	if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok {
 		for _, digest := range metadata.ArchiveDigests {
 			if digest.Algorithm == "sha1" {
-				indirectPackage, err := m.GetMavenPackageBySha(digest.Value)
+				indirectPackage, err := m.GetMavenPackageBySha(ctx, digest.Value)
 				if err != nil {
 					return nil, err
 				}

grype/matcher/java/matcher_mocks_test.go

@@ -1,6 +1,10 @@
 package java
 
 import (
+	"context"
+	"testing"
+	"time"
+
 	"github.com/anchore/grype/grype/distro"
 	"github.com/anchore/grype/grype/pkg"
 	"github.com/anchore/grype/grype/version"
@@ -49,16 +53,42 @@ func newMockProvider() *mockProvider {
 }
 
 type mockMavenSearcher struct {
-	pkg pkg.Package
+	tb   testing.TB
+	pkg  *pkg.Package
+	work *time.Duration
 }
 
-func (m mockMavenSearcher) GetMavenPackageBySha(string) (*pkg.Package, error) {
-	return &m.pkg, nil
+func newMockSearcher(tb testing.TB) mockMavenSearcher {
+	return mockMavenSearcher{
+		tb: tb,
+	}
 }
 
-func newMockSearcher(pkg pkg.Package) MavenSearcher {
-	return mockMavenSearcher{
-		pkg,
+func (m mockMavenSearcher) WithPackage(p pkg.Package) mockMavenSearcher {
+	m.pkg = &p
+	return m
+}
+
+func (m mockMavenSearcher) WithWorkDuration(duration time.Duration) mockMavenSearcher {
+	m.work = &duration
+	return m
+}
+
+func (m mockMavenSearcher) GetMavenPackageBySha(ctx context.Context, sha1 string) (*pkg.Package, error) {
+	deadline, ok := ctx.Deadline()
+
+	m.tb.Log("GetMavenPackageBySha called with deadline:", deadline, "deadline set:", ok)
+
+	if m.work != nil {
+		select {
+		case <-time.After(*m.work):
+			return m.pkg, nil
+		case <-ctx.Done():
+			// If the context is done before the sleep is over, return a context.DeadlineExceeded error
+			return m.pkg, ctx.Err()
+		}
+	} else {
+		return m.pkg, ctx.Err()
 	}
 }
 

grype/matcher/java/matcher_test.go

@@ -1,7 +1,9 @@
 package java
 
 import (
+	"context"
 	"testing"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
@@ -36,10 +38,12 @@ func TestMatcherJava_matchUpstreamMavenPackage(t *testing.T) {
 			},
 			UseCPEs: false,
 		},
-		MavenSearcher: newMockSearcher(p),
+		// no duration will return immediately with the mock data
+		MavenSearcher: newMockSearcher(t).WithPackage(p),
 	}
 	store := newMockProvider()
-	actual, _ := matcher.matchUpstreamMavenPackages(store, nil, p)
+	ctx := context.Background()
+	actual, _ := matcher.matchUpstreamMavenPackages(ctx, store, nil, p)
 
 	assert.Len(t, actual, 2, "unexpected matches count")
 
@@ -64,3 +68,38 @@ func TestMatcherJava_matchUpstreamMavenPackage(t *testing.T) {
 		t.Logf("discovered CVES: %+v", foundCVEs)
 	}
 }
+func TestMatcherJava_TestMatchUpstreamMavenPackagesTimeout(t *testing.T) {
+	p := pkg.Package{
+		ID:       pkg.ID(uuid.NewString()),
+		Name:     "org.springframework.spring-webmvc",
+		Version:  "5.1.5.RELEASE",
+		Language: syftPkg.Java,
+		Type:     syftPkg.JavaPkg,
+		Metadata: pkg.JavaMetadata{
+			ArchiveDigests: []pkg.Digest{
+				{
+					Algorithm: "sha1",
+					Value:     "236e3bfdbdc6c86629237a74f0f11414adb4e211",
+				},
+			},
+		},
+	}
+	matcher := Matcher{
+		cfg: MatcherConfig{
+			ExternalSearchConfig: ExternalSearchConfig{
+				SearchMavenUpstream: true,
+			},
+			UseCPEs: false,
+		},
+		MavenSearcher: newMockSearcher(t).WithPackage(p).WithWorkDuration(10 * time.Second),
+	}
+	store := newMockProvider()
+
+	// Create a context with a very short timeout
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
+	defer cancel()
+
+	_, err := matcher.matchUpstreamMavenPackages(ctx, store, nil, p)
+
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+}

grype/matcher/java/maven_search.go

@@ -1,6 +1,7 @@
 package java
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -14,7 +15,7 @@ import (
 // MavenSearcher is the interface that wraps the GetMavenPackageBySha method.
 type MavenSearcher interface {
 	// GetMavenPackageBySha provides an interface for building a package from maven data based on a sha1 digest
-	GetMavenPackageBySha(string) (*pkg.Package, error)
+	GetMavenPackageBySha(context.Context, string) (*pkg.Package, error)
 }
 
 // mavenSearch implements the MavenSearcher interface
@@ -37,52 +38,73 @@ type mavenAPIResponse struct {
 	} `json:"response"`
 }
 
-func (ms *mavenSearch) GetMavenPackageBySha(sha1 string) (*pkg.Package, error) {
-	req, err := http.NewRequest(http.MethodGet, ms.baseURL, nil)
-	if err != nil {
-		return nil, fmt.Errorf("unable to initialize HTTP client: %w", err)
-	}
+func (ms *mavenSearch) GetMavenPackageBySha(context context.Context, sha1 string) (*pkg.Package, error) {
+	resultChan := make(chan *pkg.Package, 1)
+	errChan := make(chan error, 1)
+	go func() {
+		req, err := http.NewRequest(http.MethodGet, ms.baseURL, nil)
+		if err != nil {
+			errChan <- fmt.Errorf("unable to initialize HTTP client: %w", err)
+			return
+		}
 
-	q := req.URL.Query()
-	q.Set("q", fmt.Sprintf(sha1Query, sha1))
-	q.Set("rows", "1")
-	q.Set("wt", "json")
-	req.URL.RawQuery = q.Encode()
+		q := req.URL.Query()
+		q.Set("q", fmt.Sprintf(sha1Query, sha1))
+		q.Set("rows", "1")
+		q.Set("wt", "json")
+		req.URL.RawQuery = q.Encode()
 
-	resp, err := ms.client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("sha1 search error: %w", err)
-	}
-	defer resp.Body.Close()
+		resp, err := ms.client.Do(req)
+		if err != nil {
+			errChan <- fmt.Errorf("sha1 search error: %w", err)
+			return
+		}
+		defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("status %s from %s", resp.Status, req.URL.String())
-	}
+		if resp.StatusCode != http.StatusOK {
+			errChan <- fmt.Errorf("status %s from %s", resp.Status, req.URL.String())
+			return
+		}
 
-	var res mavenAPIResponse
-	if err = json.NewDecoder(resp.Body).Decode(&res); err != nil {
-		return nil, fmt.Errorf("json decode error: %w", err)
-	}
+		var res mavenAPIResponse
+		if err = json.NewDecoder(resp.Body).Decode(&res); err != nil {
+			errChan <- fmt.Errorf("json decode error: %w", err)
+			return
+		}
 
-	if len(res.Response.Docs) == 0 {
-		return nil, fmt.Errorf("digest %s: %w", sha1, errors.New("no artifact found"))
-	}
+		if len(res.Response.Docs) == 0 {
+			errChan <- fmt.Errorf("digest %s: %w", sha1, errors.New("no artifact found"))
+			return
+		}
+
+		// artifacts might have the same SHA-1 digests.
+		// e.g. "javax.servlet:jstl" and "jstl:jstl"
+		docs := res.Response.Docs
+		sort.Slice(docs, func(i, j int) bool {
+			return docs[i].ID < docs[j].ID
+		})
+		d := docs[0]
 
-	// artifacts might have the same SHA-1 digests.
-	// e.g. "javax.servlet:jstl" and "jstl:jstl"
-	docs := res.Response.Docs
-	sort.Slice(docs, func(i, j int) bool {
-		return docs[i].ID < docs[j].ID
-	})
-	d := docs[0]
+		resultChan <- &pkg.Package{
+			Name:     fmt.Sprintf("%s:%s", d.GroupID, d.ArtifactID),
+			Version:  d.Version,
+			Language: syftPkg.Java,
+			Metadata: pkg.JavaMetadata{
+				PomArtifactID: d.ArtifactID,
+				PomGroupID:    d.GroupID,
+			},
+		}
+	}()
 
-	return &pkg.Package{
-		Name:     fmt.Sprintf("%s:%s", d.GroupID, d.ArtifactID),
-		Version:  d.Version,
-		Language: syftPkg.Java,
-		Metadata: pkg.JavaMetadata{
-			PomArtifactID: d.ArtifactID,
-			PomGroupID:    d.GroupID,
-		},
-	}, nil
+	select {
+	case <-context.Done():
+		// The context was canceled or its deadline was exceeded
+		return nil, context.Err()
+	case res := <-resultChan:
+		// The work finished before the context was done
+		return res, nil
+	case err := <-errChan:
+		// There was an error getting the package
+		return nil, err
+	}
 }