Skip to content

Commit

Permalink
external-sources: throttle requests to maven central to avoid being r…
Browse files Browse the repository at this point in the history
…ate limited for large sets of java depenencies

When [external-sources](https://github.com/anchore/grype?tab=readme-ov-file#external-sources) are enabled if an Image contains a large number of Java dependencies Grype can get rate limited by maven central.

This change will:
- add a rate limiter to throttle requests at 300ms per second to produce reliable results
- if a normal artifact not found error is returned by maven central the existing debug logging happens
- any other error from maven central will result in an error being logged
- adds an integration test that can be used to verify the rate limiter so we can verify against the real external api settings

Related to issue anchore#2383

Signed-off-by: James Rawlings <[email protected]>
  • Loading branch information
rawlingsj committed Jan 21, 2025
1 parent 6619733 commit f8bb4ca
Show file tree
Hide file tree
Showing 4 changed files with 740 additions and 13 deletions.
18 changes: 11 additions & 7 deletions grype/db/v5/matcher/java/matcher.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package java

import (
"context"
"fmt"
"net/http"
"strings"

v5 "github.com/anchore/grype/grype/db/v5"
"github.com/anchore/grype/grype/db/v5/search"
Expand Down Expand Up @@ -34,11 +36,8 @@ type MatcherConfig struct {

func NewJavaMatcher(cfg MatcherConfig) *Matcher {
return &Matcher{
cfg: cfg,
MavenSearcher: &mavenSearch{
client: http.DefaultClient,
baseURL: cfg.MavenBaseURL,
},
cfg: cfg,
MavenSearcher: newMavenSearch(http.DefaultClient, cfg.MavenBaseURL),
}
}

Expand All @@ -55,7 +54,10 @@ func (m *Matcher) Match(store v5.VulnerabilityProvider, d *distro.Distro, p pkg.
if m.cfg.SearchMavenUpstream {
upstreamMatches, err := m.matchUpstreamMavenPackages(store, d, p)
if err != nil {
log.Debugf("failed to match against upstream data for %s: %v", p.Name, err)
if strings.Contains(err.Error(), "no artifact found") {
log.Debugf("no upstream maven artifact found for %s", p.Name)
}
log.Errorf("failed to match against upstream data for %s: %v", p.Name, err)
} else {
matches = append(matches, upstreamMatches...)
}
Expand All @@ -76,10 +78,12 @@ func (m *Matcher) Match(store v5.VulnerabilityProvider, d *distro.Distro, p pkg.
func (m *Matcher) matchUpstreamMavenPackages(store v5.VulnerabilityProvider, d *distro.Distro, p pkg.Package) ([]match.Match, error) {
var matches []match.Match

ctx := context.Background()

if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok {
for _, digest := range metadata.ArchiveDigests {
if digest.Algorithm == "sha1" {
indirectPackage, err := m.GetMavenPackageBySha(digest.Value)
indirectPackage, err := m.GetMavenPackageBySha(ctx, digest.Value)
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit f8bb4ca

Please sign in to comment.