From 9e181bceedd967ed8c26a6fbd23f8aafd95d7987 Mon Sep 17 00:00:00 2001 From: RTann Date: Mon, 3 Jun 2024 11:09:57 -0700 Subject: [PATCH 1/6] chainguard: add chainguard and wolfi support Signed-off-by: RTann --- alpine/parser.go | 5 +- alpine/secdb_test.go | 82 ------------- chainguard/distributionscanner.go | 93 +++++++++++++++ chainguard/ecosystem.go | 27 +++++ chainguard/updater.go | 182 +++++++++++++++++++++++++++++ {alpine => updater/secdb}/secdb.go | 7 +- 6 files changed, 310 insertions(+), 86 deletions(-) delete mode 100644 alpine/secdb_test.go create mode 100644 chainguard/distributionscanner.go create mode 100644 chainguard/ecosystem.go create mode 100644 chainguard/updater.go rename {alpine => updater/secdb}/secdb.go (74%) diff --git a/alpine/parser.go b/alpine/parser.go index 015faae33..4748c1f26 100644 --- a/alpine/parser.go +++ b/alpine/parser.go @@ -9,6 +9,7 @@ import ( "github.com/quay/claircore" "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/updater/secdb" ) const ( @@ -22,7 +23,7 @@ func (u *updater) Parse(ctx context.Context, r io.ReadCloser) ([]*claircore.Vuln zlog.Info(ctx).Msg("starting parse") defer r.Close() - var db SecurityDB + var db secdb.SecurityDB if err := json.NewDecoder(r).Decode(&db); err != nil { return nil, err } @@ -30,7 +31,7 @@ func (u *updater) Parse(ctx context.Context, r io.ReadCloser) ([]*claircore.Vuln } // parse parses the alpine SecurityDB -func (u *updater) parse(ctx context.Context, sdb *SecurityDB) ([]*claircore.Vulnerability, error) { +func (u *updater) parse(ctx context.Context, sdb *secdb.SecurityDB) ([]*claircore.Vulnerability, error) { out := []*claircore.Vulnerability{} for _, pkg := range sdb.Packages { if err := ctx.Err(); err != nil { diff --git a/alpine/secdb_test.go b/alpine/secdb_test.go deleted file mode 100644 index f0e3cb749..000000000 --- a/alpine/secdb_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package alpine - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" -) - -var v3_10CommunityTruncatedSecDB = SecurityDB{ - Distroversion: "v3.10", - Reponame: "community", - Urlprefix: "http://dl-cdn.alpinelinux.org/alpine", - Apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-{{pkg.ver}}.apk", - Packages: []Package{ - { - Pkg: Details{ - Name: "botan", - Secfixes: map[string][]string{ - "2.9.0-r0": {"CVE-2018-20187"}, - "2.7.0-r0": {"CVE-2018-12435"}, - "2.6.0-r0": {"CVE-2018-9860"}, - "2.5.0-r0": {"CVE-2018-9127"}, - }, - }, - }, - { - Pkg: Details{ - Name: "cfengine", - Secfixes: map[string][]string{ - "3.12.2-r0": {"CVE-2019-9929"}, - }, - }, - }, - { - Pkg: Details{ - Name: "chicken", - Secfixes: map[string][]string{ - "4.12.0-r3": {"CVE-2017-6949"}, - "4.12.0-r2": {"CVE-2017-9334"}, - "4.11.1-r0": {"CVE-2016-6830", "CVE-2016-6831"}, - }, - }, - }, - }, -} - -func TestSecDBParse(t *testing.T) { - var table = []struct { - testFile string - expected SecurityDB - }{ - { - testFile: "fetch/v3.10/community.json", - expected: v3_10CommunityTruncatedSecDB, - }, - } - - for _, test := range table { - path := filepath.Join("testdata", test.testFile) - want := test.expected - t.Run(test.testFile, func(t *testing.T) { - t.Parallel() - - f, err := os.Open(path) - if err != nil { - t.Fatalf("failed to open test data: %v", path) - } - - var got SecurityDB - if err := json.NewDecoder(f).Decode(&got); err != nil { - t.Fatalf("failed to parse file contents into sec db: %v", err) - } - if !cmp.Equal(got, want) { - t.Log("security databases were not equal:") - t.Error(cmp.Diff(got, want)) - } - }) - } -} diff --git a/chainguard/distributionscanner.go b/chainguard/distributionscanner.go new file mode 100644 index 000000000..51bd5db57 --- /dev/null +++ b/chainguard/distributionscanner.go @@ -0,0 +1,93 @@ +package chainguard + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/quay/zlog" + "io/fs" + "runtime/trace" + + "github.com/quay/claircore" + "github.com/quay/claircore/indexer" + "github.com/quay/claircore/osrelease" +) + +const ( + scannerName = "chainguard" + scannerVersion = "1" + scannerKind = "distribution" + + chainguard = `chainguard` + wolfi = `wolfi` +) + +var ( + _ indexer.DistributionScanner = (*DistributionScanner)(nil) + _ indexer.VersionedScanner = (*DistributionScanner)(nil) +) + +// DistributionScanner attempts to discover if a layer +// displays characteristics of a chainguard or wolfi distribution. +type DistributionScanner struct{} + +// Name implements scanner.VersionedScanner. +func (*DistributionScanner) Name() string { return scannerName } + +// Version implements scanner.VersionedScanner. +func (*DistributionScanner) Version() string { return scannerVersion } + +// Kind implements scanner.VersionedScanner. +func (*DistributionScanner) Kind() string { return scannerKind } + +// Scan will inspect the layer for an os-release +// and determine if it represents a chainguard or wolfi release. +// +// If the file is not found, or the file does not represent a chainguard nor wolfi release, +// (nil, nil) is returned. +func (s *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) { + defer trace.StartRegion(ctx, "Scanner.Scan").End() + ctx = zlog.ContextWithValues(ctx, + "component", "chainguard/DistributionScanner.Scan", + "version", s.Version(), + "layer", l.Hash.String()) + zlog.Debug(ctx).Msg("start") + defer zlog.Debug(ctx).Msg("done") + + sys, err := l.FS() + if err != nil { + return nil, fmt.Errorf("chainguard: unable to open layer: %w", err) + } + + b, err := fs.ReadFile(sys, osrelease.Path) + switch { + case errors.Is(err, nil): + m, err := osrelease.Parse(ctx, bytes.NewReader(b)) + if err != nil { + return nil, err + } + + switch id := m[`ID`]; id { + case chainguard, wolfi: + return []*claircore.Distribution{ + { + Name: m[`NAME`], + DID: id, + // Neither chainguard nor wolfi images are considered to be "versioned". + // Explicitly set the version to the empty string for clarity. + Version: "", + PrettyName: m[`PRETTY_NAME`], + }, + }, nil + default: + // This is neither chainguard nor wolfi. + return nil, nil + } + case errors.Is(err, fs.ErrNotExist): + // os-release file must exist in chainguard and wolfi images. + return nil, nil + default: + return nil, err + } +} diff --git a/chainguard/ecosystem.go b/chainguard/ecosystem.go new file mode 100644 index 000000000..b6dc5c83f --- /dev/null +++ b/chainguard/ecosystem.go @@ -0,0 +1,27 @@ +package chainguard + +import ( + "context" + + "github.com/quay/claircore/apk" + "github.com/quay/claircore/indexer" + "github.com/quay/claircore/linux" +) + +// NewEcosystem provides the set of scanners and coalescers for the chainguard ecosystem +func NewEcosystem(_ context.Context) *indexer.Ecosystem { + return &indexer.Ecosystem{ + PackageScanners: func(ctx context.Context) ([]indexer.PackageScanner, error) { + return []indexer.PackageScanner{&apk.Scanner{}}, nil + }, + DistributionScanners: func(ctx context.Context) ([]indexer.DistributionScanner, error) { + return []indexer.DistributionScanner{&DistributionScanner{}}, nil + }, + RepositoryScanners: func(ctx context.Context) ([]indexer.RepositoryScanner, error) { + return []indexer.RepositoryScanner{}, nil + }, + Coalescer: func(ctx context.Context) (indexer.Coalescer, error) { + return linux.NewCoalescer(), nil + }, + } +} diff --git a/chainguard/updater.go b/chainguard/updater.go new file mode 100644 index 000000000..503758324 --- /dev/null +++ b/chainguard/updater.go @@ -0,0 +1,182 @@ +package chainguard + +import ( + "context" + "fmt" + "github.com/quay/claircore" + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/zlog" + "io" + "net/http" +) + +//doc:url updater +const ( + chainguardURL = `https://packages.cgr.dev/chainguard/security.json` + wolfiURL = `https://packages.wolfi.dev/os/security.json` +) + +var ( + _ driver.UpdaterSetFactory = (*Factory)(nil) + _ driver.Configurable = (*Factory)(nil) + _ driver.Updater = (*updater)(nil) + _ driver.Configurable = (*updater)(nil) +) + +// Factory is an UpdaterSetFactory for ingesting Chainguard and Wolfi SecDBs. +// +// Factory expects to be able to discover a directory layout like the one at [https://secdb.alpinelinux.org/] at the configured URL. +// More explictly, it expects: +// - a "last-update" file with opaque contents that change when any constituent database changes +// - contiguously numbered directories with the name "v$maj.$min" starting with "maj" as "3" and "min" as at most "3" +// - JSON files inside those directories named "main.json" or "community.json" +// +// The [Configure] method must be called before the [UpdaterSet] method. +type Factory struct { + client *http.Client + + chainguardURL string + chainguardETag string + + wolfiURL string + wolfiETag string +} + +// NewFactory returns a constructed Factory. +// +// [Configure] must still be called before [UpdaterSet]. +func NewFactory(_ context.Context) (*Factory, error) { + return &Factory{ + chainguardURL: chainguardURL, + wolfiURL: wolfiURL, + }, nil +} + +// FactoryConfig is the configuration accepted by the Factory. +type FactoryConfig struct { + // ChainguardURL indicates the URL for the Chainguard SecDB. + ChainguardURL string `json:"chainguard_url" yaml:"chainguard_url"` + // WolfiURL indicates the URL for the Wolfi SecDB. + WolfiURL string `json:"wolfi_url" yaml:"wolfi_url"` +} + +// Configure implements driver.Configurable. +func (f *Factory) Configure(_ context.Context, cf driver.ConfigUnmarshaler, c *http.Client) error { + f.client = c + var cfg FactoryConfig + if err := cf(&cfg); err != nil { + return err + } + if cfg.ChainguardURL != "" { + f.chainguardURL = cfg.ChainguardURL + } + if cfg.WolfiURL != "" { + f.wolfiURL = cfg.WolfiURL + } + return nil +} + +func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { + ctx = zlog.ContextWithValues(ctx, "component", "chainguard/Factory.UpdaterSet") + + s := driver.NewUpdaterSet() + + add, err := addToUpdaterSet(ctx, f.client, f.chainguardURL, f.chainguardETag) + if err != nil { + return s, err + } + if add { + s.Add(&updater{ + name: "chainguard", + url: f.chainguardURL, + }) + } + + add, err = addToUpdaterSet(ctx, f.client, f.wolfiURL, f.wolfiURL) + if err != nil { + return s, err + } + if add { + s.Add(&updater{ + name: "wolfi", + url: f.wolfiURL, + }) + } + + return s, nil +} + +func addToUpdaterSet(ctx context.Context, client *http.Client, url, etag string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + return false, fmt.Errorf("chainguard: unable to construct request to %q: %w", url, err) + } + if etag != "" { + req.Header.Set("If-None-Match", etag) + } + + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("chainguard: error requesting %q: %w", url, err) + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusNotModified: + return false, nil + case http.StatusOK: + return true, nil + default: + return false, fmt.Errorf("chainguard: unexpected status requesting %q: %s", url, res.Status) + } +} + +type updater struct { + name string + client *http.Client + url string +} + +// UpdaterConfig is the configuration accepted by Chainguard and Wolfi updaters. +// +// By convention, this should be in a map called "-updater". +// For example, "alpine-main-v3.12-updater". +// +// If a SecDB JSON file is not found at the proper place by [Factory.UpdaterSet], this configuration will not be consulted. +type UpdaterConfig struct { + // URL overrides any discovered URL for the JSON file. + ChainguardURL string `json:"chainguard_url" yaml:"chainguard_url"` + WolfiURL string `json:"wolfi_url" yaml:"wolfi_url"` +} + +// Configure implements driver.Configurable. +func (u *updater) Configure(ctx context.Context, f driver.ConfigUnmarshaler, c *http.Client) error { + var cfg UpdaterConfig + if err := f(&cfg); err != nil { + return err + } + if cfg.URL != "" { + u.url = cfg.URL + zlog.Info(ctx). + Str("component", "alpine/Updater.Configure"). + Str("updater", u.Name()). + Msg("configured url") + } + u.client = c + return nil +} + +func (u updater) Name() string { + //TODO implement me + panic("implement me") +} + +func (u updater) Fetch(ctx context.Context, fingerprint driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) { + //TODO implement me + panic("implement me") +} + +func (u updater) Parse(ctx context.Context, contents io.ReadCloser) ([]*claircore.Vulnerability, error) { + //TODO implement me + panic("implement me") +} diff --git a/alpine/secdb.go b/updater/secdb/secdb.go similarity index 74% rename from alpine/secdb.go rename to updater/secdb/secdb.go index bf6c68e42..0663b16cd 100644 --- a/alpine/secdb.go +++ b/updater/secdb/secdb.go @@ -1,4 +1,7 @@ -package alpine +// Package secdb defines the common structs used in secdbs. +// +// Specifically, this is common between Alpine, Chainguard, and Wolfi. +package secdb // Details define a package's name and relevant security fixes included in a // given version. @@ -16,7 +19,7 @@ type Package struct { // SecurityDB is the security database structure. type SecurityDB struct { - Distroversion string `json:"distroversion"` + Distroversion string `json:"distroversion"` // Alpine, only Reponame string `json:"reponame"` Urlprefix string `json:"urlprefix"` Apkurl string `json:"apkurl"` From f6f21d1bcc08dbf2704ff522235af1a2f35fc5b2 Mon Sep 17 00:00:00 2001 From: RTann Date: Mon, 24 Feb 2025 13:46:40 -0800 Subject: [PATCH 2/6] more stuff Signed-off-by: RTann --- chainguard/distributionscanner.go | 40 +- chainguard/doc.go | 7 + chainguard/ecosystem.go | 2 +- chainguard/fetcher.go | 83 ++++ chainguard/matcher.go | 81 ++++ chainguard/parser.go | 72 +++ chainguard/release.go | 27 ++ chainguard/updater.go | 29 +- enricher/kev/kev.go | 272 +++++++++++ enricher/kev/kev_test.go | 440 ++++++++++++++++++ .../known_exploited_vulnerabilities.json | 40 ++ enricher/kev/types.go | 34 ++ 12 files changed, 1081 insertions(+), 46 deletions(-) create mode 100644 chainguard/doc.go create mode 100644 chainguard/fetcher.go create mode 100644 chainguard/matcher.go create mode 100644 chainguard/parser.go create mode 100644 chainguard/release.go create mode 100644 enricher/kev/kev.go create mode 100644 enricher/kev/kev_test.go create mode 100644 enricher/kev/testdata/known_exploited_vulnerabilities.json create mode 100644 enricher/kev/types.go diff --git a/chainguard/distributionscanner.go b/chainguard/distributionscanner.go index 51bd5db57..01c91c6f0 100644 --- a/chainguard/distributionscanner.go +++ b/chainguard/distributionscanner.go @@ -19,8 +19,8 @@ const ( scannerVersion = "1" scannerKind = "distribution" - chainguard = `chainguard` - wolfi = `wolfi` + chainguardID = `chainguard` + wolfiID = `wolfi` ) var ( @@ -63,31 +63,25 @@ func (s *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]* b, err := fs.ReadFile(sys, osrelease.Path) switch { case errors.Is(err, nil): - m, err := osrelease.Parse(ctx, bytes.NewReader(b)) - if err != nil { - return nil, err - } - - switch id := m[`ID`]; id { - case chainguard, wolfi: - return []*claircore.Distribution{ - { - Name: m[`NAME`], - DID: id, - // Neither chainguard nor wolfi images are considered to be "versioned". - // Explicitly set the version to the empty string for clarity. - Version: "", - PrettyName: m[`PRETTY_NAME`], - }, - }, nil - default: - // This is neither chainguard nor wolfi. - return nil, nil - } case errors.Is(err, fs.ErrNotExist): // os-release file must exist in chainguard and wolfi images. return nil, nil default: return nil, err } + + m, err := osrelease.Parse(ctx, bytes.NewReader(b)) + if err != nil { + return nil, err + } + + switch id := m[`ID`]; id { + case chainguardID: + return []*claircore.Distribution{chainguardDist}, nil + case wolfiID: + return []*claircore.Distribution{wolfiDist}, nil + default: + // This is neither chainguard nor wolfi. + return nil, nil + } } diff --git a/chainguard/doc.go b/chainguard/doc.go new file mode 100644 index 000000000..4b5e57d65 --- /dev/null +++ b/chainguard/doc.go @@ -0,0 +1,7 @@ +// Package chainguard implements everything required to index chainguard-based and wolfi-based container images +// and match their respective packages to related vulnerabilities. +// +// The implementation is based on the [documentation] provided upstream. +// +// [documentation] https://github.com/chainguard-dev/vulnerability-scanner-support/blob/main/docs/scanning_implementation.md +package chainguard diff --git a/chainguard/ecosystem.go b/chainguard/ecosystem.go index b6dc5c83f..86e056cf0 100644 --- a/chainguard/ecosystem.go +++ b/chainguard/ecosystem.go @@ -8,7 +8,7 @@ import ( "github.com/quay/claircore/linux" ) -// NewEcosystem provides the set of scanners and coalescers for the chainguard ecosystem +// NewEcosystem provides the set of scanners and coalescers for the chainguard/wolfi ecosystem. func NewEcosystem(_ context.Context) *indexer.Ecosystem { return &indexer.Ecosystem{ PackageScanners: func(ctx context.Context) ([]indexer.PackageScanner, error) { diff --git a/chainguard/fetcher.go b/chainguard/fetcher.go new file mode 100644 index 000000000..d01e62801 --- /dev/null +++ b/chainguard/fetcher.go @@ -0,0 +1,83 @@ +package chainguard + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/quay/zlog" + + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/pkg/tmp" +) + +func (u *updater) Fetch(ctx context.Context, hint driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) { + ctx = zlog.ContextWithValues(ctx, "component", "chainguard/Updater.Fetch") + + zlog.Info(ctx).Str("database", u.url).Msg("starting fetch") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.url, nil) + if err != nil { + return nil, hint, fmt.Errorf("chainguard: unable to construct request: %w", err) + } + + if hint != "" { + zlog.Debug(ctx). + Str("hint", string(hint)). + Msg("using hint") + req.Header.Set("if-none-match", string(hint)) + } + + res, err := u.client.Do(req) + if err != nil { + return nil, hint, fmt.Errorf("chainguard: error making request: %w", err) + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + if t := string(hint); t == "" || t != res.Header.Get("etag") { + break + } + fallthrough + case http.StatusNotModified: + zlog.Info(ctx).Msg("database unchanged since last fetch") + return nil, hint, driver.Unchanged + default: + return nil, hint, fmt.Errorf("chainguard: http response error: %s %d", res.Status, res.StatusCode) + } + zlog.Debug(ctx).Msg("successfully requested database") + + tf, err := tmp.NewFile("", u.Name()+".") + if err != nil { + return nil, hint, fmt.Errorf("chainguard: unable to open tempfile: %w", err) + } + zlog.Debug(ctx). + Str("name", tf.Name()). + Msg("created tempfile") + var success bool + defer func() { + if !success { + if err := tf.Close(); err != nil { + zlog.Warn(ctx).Err(err).Msg("unable to close spool") + } + } + }() + + var r io.Reader = res.Body + if _, err := io.Copy(tf, r); err != nil { + return nil, hint, fmt.Errorf("chainguard: unable to copy resp body to tempfile: %w", err) + } + if n, err := tf.Seek(0, io.SeekStart); err != nil || n != 0 { + return nil, hint, fmt.Errorf("chainguard: unable to seek database to start: at %d, %v", n, err) + } + zlog.Debug(ctx).Msg("decompressed and buffered database") + + success = true + hint = driver.Fingerprint(res.Header.Get("etag")) + zlog.Debug(ctx). + Str("hint", string(hint)). + Msg("using new hint") + + return tf, hint, nil +} diff --git a/chainguard/matcher.go b/chainguard/matcher.go new file mode 100644 index 000000000..d64d7ca19 --- /dev/null +++ b/chainguard/matcher.go @@ -0,0 +1,81 @@ +package chainguard + +import ( + "context" + + version "github.com/knqyf263/go-apk-version" + "github.com/quay/claircore" + "github.com/quay/claircore/libvuln/driver" +) + +var ( + ChainguardMatcher = &matcher{"chainguard"} + WolfiMatcher = &matcher{"wolfi"} +) + +var _ driver.Matcher = (*matcher)(nil) + +// Matcher implements driver.Matcher for Chainguard and Wolfi containers. +type matcher struct { + name string +} + +// Name implements driver.Matcher. +func (m *matcher) Name() string { + return m.name + "-matcher" +} + +// Filter implements driver.Matcher. +func (m *matcher) Filter(record *claircore.IndexRecord) bool { + if record.Distribution == nil { + return false + } + + switch { + case record.Distribution.DID == m.name: + return true + case record.Distribution.Name == m.name: + return true + default: + return false + } +} + +// Query implements driver.Matcher. +func (*matcher) Query() []driver.MatchConstraint { + return []driver.MatchConstraint{ + driver.DistributionDID, + driver.DistributionName, + driver.DistributionPrettyName, + } +} + +// Vulnerable implements driver.Matcher. +func (*matcher) Vulnerable(_ context.Context, record *claircore.IndexRecord, vuln *claircore.Vulnerability) (bool, error) { + if vuln.FixedInVersion == "" { + return true, nil + } + + // Version "0" tracks false-positives, and it indicates the package is not affected by the vulnerability. + // See the following for more information: + // https://github.com/chainguard-dev/vulnerability-scanner-support/blob/main/docs/scanning_implementation.md#the-meaning-of-version-0 + if vuln.FixedInVersion == "0" { + return false, nil + } + + pkgVersion, err := version.NewVersion(record.Package.Version) + if err != nil { + return false, nil + } + + fixedInVersion, err := version.NewVersion(vuln.FixedInVersion) + if err != nil { + return false, nil + } + + if pkgVersion.LessThan(fixedInVersion) { + return true, nil + } + + return false, nil +} diff --git a/chainguard/parser.go b/chainguard/parser.go new file mode 100644 index 000000000..eb73c4f52 --- /dev/null +++ b/chainguard/parser.go @@ -0,0 +1,72 @@ +package chainguard + +import ( + "context" + "encoding/json" + "fmt" + "github.com/quay/zlog" + "io" + + "github.com/quay/claircore" + "github.com/quay/claircore/updater/secdb" +) + +const urlPrefix = "https://images.chainguard.dev/security/" + +func (u *updater) Parse(ctx context.Context, r io.ReadCloser) ([]*claircore.Vulnerability, error) { + ctx = zlog.ContextWithValues(ctx, "component", "chainguard/Updater.Parse") + zlog.Info(ctx).Msg("starting parse") + defer r.Close() + + var db secdb.SecurityDB + if err := json.NewDecoder(r).Decode(&db); err != nil { + return nil, err + } + return u.parse(ctx, &db) +} + +// parse parses the alpine SecurityDB +func (u *updater) parse(ctx context.Context, sdb *secdb.SecurityDB) ([]*claircore.Vulnerability, error) { + var dist *claircore.Distribution + switch u.Name() { + case "chainguard-updater": + dist = chainguardDist + case "wolfi-updater": + dist = wolfiDist + } + if dist == nil { + return nil, fmt.Errorf("chainguard: no distribution found for %s", u.Name()) + } + out := []*claircore.Vulnerability{} + for _, pkg := range sdb.Packages { + if err := ctx.Err(); err != nil { + return nil, ctx.Err() + } + partial := claircore.Vulnerability{ + Updater: u.Name(), + NormalizedSeverity: claircore.Unknown, + Package: &claircore.Package{ + Name: pkg.Pkg.Name, + Kind: claircore.SOURCE, + }, + Dist: dist, + } + out = append(out, unpackSecFixes(partial, pkg.Pkg.Secfixes)...) + } + return out, nil +} + +// unpackSecFixes takes a map of secFixes and creates a claircore.Vulnerability for each all CVEs present. +func unpackSecFixes(partial claircore.Vulnerability, secFixes map[string][]string) []*claircore.Vulnerability { + out := []*claircore.Vulnerability{} + for fixedIn, ids := range secFixes { + for _, id := range ids { + v := partial + v.Name = id + v.FixedInVersion = fixedIn + v.Links = urlPrefix + id + out = append(out, &v) + } + } + return out +} diff --git a/chainguard/release.go b/chainguard/release.go new file mode 100644 index 000000000..2da55be11 --- /dev/null +++ b/chainguard/release.go @@ -0,0 +1,27 @@ +package chainguard + +import ( + "github.com/quay/claircore" +) + +var chainguardDist = &claircore.Distribution{ + Name: "chainguard", + DID: "chainguard", + // Chainguard images are not versioned. + // Explicitly set the version to the empty string for clarity. + // See https://github.com/chainguard-dev/vulnerability-scanner-support/blob/main/docs/scanning_implementation.md#chainguards-distros-are-not-versioned + // for more information. + Version: "", + PrettyName: "Chainguard", +} + +var wolfiDist = &claircore.Distribution{ + Name: "wolfi", + DID: "wolfi", + // Wolfi images are not versioned. + // Explicitly set the version to the empty string for clarity. + // See https://github.com/chainguard-dev/vulnerability-scanner-support/blob/main/docs/scanning_implementation.md#chainguards-distros-are-not-versioned + // for more information. + Version: "", + PrettyName: "Wolfi", +} diff --git a/chainguard/updater.go b/chainguard/updater.go index 503758324..181b8d7c8 100644 --- a/chainguard/updater.go +++ b/chainguard/updater.go @@ -3,10 +3,8 @@ package chainguard import ( "context" "fmt" - "github.com/quay/claircore" "github.com/quay/claircore/libvuln/driver" "github.com/quay/zlog" - "io" "net/http" ) @@ -87,7 +85,7 @@ func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { } if add { s.Add(&updater{ - name: "chainguard", + name: "chainguard-updater", url: f.chainguardURL, }) } @@ -98,7 +96,7 @@ func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { } if add { s.Add(&updater{ - name: "wolfi", + name: "wolfi-updater", url: f.wolfiURL, }) } @@ -139,14 +137,12 @@ type updater struct { // UpdaterConfig is the configuration accepted by Chainguard and Wolfi updaters. // -// By convention, this should be in a map called "-updater". -// For example, "alpine-main-v3.12-updater". +// By convention, this should be in a map called "chainguard-updater" or "wolfi-updater". // // If a SecDB JSON file is not found at the proper place by [Factory.UpdaterSet], this configuration will not be consulted. type UpdaterConfig struct { // URL overrides any discovered URL for the JSON file. - ChainguardURL string `json:"chainguard_url" yaml:"chainguard_url"` - WolfiURL string `json:"wolfi_url" yaml:"wolfi_url"` + URL string `json:"url" yaml:"url"` } // Configure implements driver.Configurable. @@ -158,7 +154,7 @@ func (u *updater) Configure(ctx context.Context, f driver.ConfigUnmarshaler, c * if cfg.URL != "" { u.url = cfg.URL zlog.Info(ctx). - Str("component", "alpine/Updater.Configure"). + Str("component", "chainguard/Updater.Configure"). Str("updater", u.Name()). Msg("configured url") } @@ -166,17 +162,6 @@ func (u *updater) Configure(ctx context.Context, f driver.ConfigUnmarshaler, c * return nil } -func (u updater) Name() string { - //TODO implement me - panic("implement me") -} - -func (u updater) Fetch(ctx context.Context, fingerprint driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) { - //TODO implement me - panic("implement me") -} - -func (u updater) Parse(ctx context.Context, contents io.ReadCloser) ([]*claircore.Vulnerability, error) { - //TODO implement me - panic("implement me") +func (u *updater) Name() string { + return u.name } diff --git a/enricher/kev/kev.go b/enricher/kev/kev.go new file mode 100644 index 000000000..65e12d9df --- /dev/null +++ b/enricher/kev/kev.go @@ -0,0 +1,272 @@ +// Package kev provides a CISA Known Exploited Vulnerabilities enricher. +package kev + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strings" + + "github.com/quay/zlog" + + "github.com/quay/claircore" + "github.com/quay/claircore/enricher" + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/pkg/tmp" +) + +var ( + _ driver.Enricher = (*Enricher)(nil) + _ driver.EnrichmentUpdater = (*Enricher)(nil) + + defaultFeed *url.URL +) + +const ( + // Type is the type of data returned from the Enricher's Enrich method. + Type = `message/vnd.clair.map.vulnerability; enricher=clair.kev schema=https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities_schema.json` + // DefaultFeed is the default place to look for the CISA Known Exploited Vulnerabilities feed. + // + //doc:url updater + DefaultFeed = `https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json` + + // This appears above and must be the same. + name = `clair.kev` +) + +func init() { + var err error + defaultFeed, err = url.Parse(DefaultFeed) + if err != nil { + panic(err) + } +} + +// Enricher provides exploit data as enrichments to a VulnerabilityReport. +// +// Configure must be called before any other methods. +type Enricher struct { + driver.NoopUpdater + c *http.Client + feed *url.URL +} + +// Config is the configuration for Enricher. +type Config struct { + Feed *string `json:"feed_root" yaml:"feed"` +} + +// Configure implements driver.Configurable. +func (e *Enricher) Configure(_ context.Context, f driver.ConfigUnmarshaler, c *http.Client) error { + var cfg Config + e.c = c + if err := f(&cfg); err != nil { + return err + } + e.feed = defaultFeed + if cfg.Feed != nil { + if !strings.HasSuffix(*cfg.Feed, ".json") { + return fmt.Errorf("URL not pointing to JSON: %q", *cfg.Feed) + } + u, err := url.Parse(*cfg.Feed) + if err != nil { + return err + } + e.feed = u + } + return nil +} + +// Name implements driver.Enricher and driver.EnrichmentUpdater. +func (*Enricher) Name() string { return name } + +// FetchEnrichment implements driver.EnrichmentUpdater. +func (e *Enricher) FetchEnrichment(ctx context.Context, hint driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) { + ctx = zlog.ContextWithValues(ctx, "component", "enricher/kev/Enricher/FetchEnrichment") + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, e.feed.String(), nil) + if err != nil { + return nil, hint, err + } + if hint != "" { + // Note: Though the default URL returns an etag, the server does not seem to respond + // to the If-None-Match header. It seems like it does respond to If-Modified-Since, though, + // so the timestamp is used as the hint. + req.Header.Set("If-Modified-Since", string(hint)) + } + res, err := e.c.Do(req) + if err != nil { + return nil, hint, err + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + if t := string(hint); t == "" || t != res.Header.Get("Last-Modified") { + break + } + fallthrough + case http.StatusNotModified: + zlog.Info(ctx).Msg("database unchanged since last fetch") + return nil, hint, driver.Unchanged + default: + return nil, hint, fmt.Errorf("http response error: %s %d", res.Status, res.StatusCode) + } + zlog.Debug(ctx).Msg("successfully requested database") + + out, err := tmp.NewFile("", "kev.") + if err != nil { + return nil, hint, err + } + var success bool + defer func() { + if !success { + if err := out.Close(); err != nil { + zlog.Warn(ctx).Err(err).Msg("unable to close spool") + } + } + }() + + // When originally created, the file was around 1.1MB, so + // it seems like a good idea to buffer. + buf := bufio.NewReader(res.Body) + _, err = io.Copy(out, buf) + if err != nil { + return nil, hint, fmt.Errorf("failed to read enrichment: %w", err) + } + + if _, err := out.Seek(0, io.SeekStart); err != nil { + return nil, hint, fmt.Errorf("unable to reset spool: %w", err) + } + + success = true + hint = driver.Fingerprint(res.Header.Get("Last-Modified")) + zlog.Debug(ctx). + Str("hint", string(hint)). + Msg("using new hint") + + return out, hint, nil +} + +// ParseEnrichment implements driver.EnrichmentUpdater. +func (e *Enricher) ParseEnrichment(ctx context.Context, rc io.ReadCloser) ([]driver.EnrichmentRecord, error) { + ctx = zlog.ContextWithValues(ctx, "component", "enricher/kev/Enricher/ParseEnrichment") + + var root Root + buf := bufio.NewReader(rc) + if err := json.NewDecoder(buf).Decode(&root); err != nil { + return nil, fmt.Errorf("failed to parse enrichment: %w", err) + } + + // The self-declared count is probably pretty accurate. + // As of writing this, the count is 1278, so it's rather small. + recs := make([]driver.EnrichmentRecord, 0, root.Count) + for _, vuln := range root.Vulnerabilities { + entry := Entry{ + CVE: vuln.CVEID, + VulnerabilityName: vuln.VulnerabilityName, + CatalogVersion: root.CatalogVersion, + DateAdded: vuln.DateAdded, + ShortDescription: vuln.ShortDescription, + RequiredAction: vuln.RequiredAction, + DueDate: vuln.DueDate, + KnownRansomwareCampaignUse: vuln.KnownRansomwareCampaignUse, + } + enrichment, err := json.Marshal(&entry) + if err != nil { + return nil, fmt.Errorf("failed to encode enrichment: %w", err) + } + + recs = append(recs, driver.EnrichmentRecord{ + Tags: []string{vuln.CVEID}, + Enrichment: enrichment, + }) + } + + return recs, nil +} + +// Enrich implements driver.Enricher. +func (e *Enricher) Enrich(ctx context.Context, g driver.EnrichmentGetter, r *claircore.VulnerabilityReport) (string, []json.RawMessage, error) { + ctx = zlog.ContextWithValues(ctx, "component", "enricher/kev/Enricher/Enrich") + + m := make(map[string][]json.RawMessage) + erCache := make(map[string][]driver.EnrichmentRecord) + + for id, v := range r.Vulnerabilities { + t := make(map[string]struct{}) + ctx := zlog.ContextWithValues(ctx, "vuln", v.Name) + + for _, elem := range []string{ + v.Description, + v.Name, + v.Links, + } { + // Check if the element is non-empty before running the regex + if elem == "" { + zlog.Debug(ctx).Str("element", elem).Msg("skipping empty element") + continue + } + + matches := enricher.CVERegexp.FindAllString(elem, -1) + if len(matches) == 0 { + zlog.Debug(ctx).Str("element", elem).Msg("no CVEs found in element") + continue + } + for _, m := range matches { + t[m] = struct{}{} + } + } + + // Skip if no CVEs were found + if len(t) == 0 { + zlog.Debug(ctx).Msg("no CVEs found in vulnerability metadata") + continue + } + + ts := make([]string, 0, len(t)) + for m := range t { + ts = append(ts, m) + } + slices.Sort(ts) + + cveKey := strings.Join(ts, "_") + + rec, ok := erCache[cveKey] + if !ok { + var err error + rec, err = g.GetEnrichment(ctx, ts) + if err != nil { + return "", nil, err + } + erCache[cveKey] = rec + } + + zlog.Debug(ctx).Int("count", len(rec)).Msg("found records") + + // Skip if no enrichment records are found + if len(rec) == 0 { + zlog.Debug(ctx).Strs("cve", ts).Msg("no enrichment records found for CVEs") + continue + } + + for _, r := range rec { + m[id] = append(m[id], r.Enrichment) + } + } + + if len(m) == 0 { + return Type, nil, nil + } + + b, err := json.Marshal(m) + if err != nil { + return Type, nil, err + } + return Type, []json.RawMessage{b}, nil +} diff --git a/enricher/kev/kev_test.go b/enricher/kev/kev_test.go new file mode 100644 index 000000000..c924e77c4 --- /dev/null +++ b/enricher/kev/kev_test.go @@ -0,0 +1,440 @@ +package kev + +import ( + "context" + "encoding/json" + "errors" + "github.com/google/go-cmp/cmp" + "github.com/quay/zlog" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "testing" + + "github.com/quay/claircore" + "github.com/quay/claircore/libvuln/driver" +) + +// Define a static Last-Modified for testing purposes +const lastModified = `Mon, 24 Feb 2025 17:55:31 GMT` + +func noopConfig(_ interface{}) error { return nil } + +func TestConfigure(t *testing.T) { + t.Parallel() + ctx := zlog.Test(context.Background(), t) + + type configTestcase struct { + Config func(interface{}) error + Check func(*testing.T, error) + Name string + } + + tt := []configTestcase{ + { + Name: "None", // No configuration provided, should use default + Check: func(t *testing.T, err error) { + if err != nil { + t.Errorf("unexpected error: %v", err) + } + }, + }, + { + Name: "Not OK", // URL without .json is invalid + Config: func(i interface{}) error { + cfg := i.(*Config) + s := "https://example.com/exploits.csv" + cfg.Feed = &s + return nil + }, + Check: func(t *testing.T, err error) { + if err == nil { + t.Errorf("expected invalid URL error, but got none: %v", err) + } + }, + }, + + { + Name: "UnmarshalError", // Expected error on unmarshaling + Config: func(_ interface{}) error { return errors.New("expected error") }, + Check: func(t *testing.T, err error) { + if err == nil { + t.Error("expected unmarshal error, but got none") + } + }, + }, + { + Name: "BadURL", // Malformed URL in URL + Config: func(i interface{}) error { + cfg := i.(*Config) + s := "http://[notaurl:/" + cfg.Feed = &s + return nil + }, + Check: func(t *testing.T, err error) { + if err == nil { + t.Error("expected URL parse error, but got none") + } + }, + }, + { + Name: "ValidURL", // Proper .json URL in URL + Config: func(i interface{}) error { + cfg := i.(*Config) + s := "https://www.example.com/sites/default/files/feeds/known_exploited_vulnerabilities.json" + cfg.Feed = &s + return nil + }, + Check: func(t *testing.T, err error) { + if err != nil { + t.Errorf("unexpected error with .json URL: %v", err) + } + }, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + e := &Enricher{} + ctx := zlog.Test(ctx, t) + f := tc.Config + if f == nil { + f = noopConfig + } + err := e.Configure(ctx, f, nil) + if tc.Check == nil { + if err != nil { + t.Errorf("unexpected err: %v", err) + } + return + } + tc.Check(t, err) + }) + } +} + +func mockServer(t *testing.T) *httptest.Server { + const root = `testdata/` + lastModifiedTime, err := http.ParseTime(lastModified) + if err != nil { + t.Fatalf("unable to parse last modified time: %v", err) + } + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch path.Ext(r.URL.Path) { + case ".json": // only JSON is supported + w.Header().Set("Last-Modified", lastModified) + + f, err := os.Open(filepath.Join(root, "known_exploited_vulnerabilities.json")) + if err != nil { + t.Errorf("open failed: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + defer f.Close() + + ims := r.Header.Get("If-Modified-Since") + if ims != "" { + imsTime, err := http.ParseTime(ims) + if err != nil { + t.Errorf("parse time failed: %v", err) + } + if lastModifiedTime.Before(imsTime) { + w.WriteHeader(http.StatusNotModified) + return + } + } + + _, err = io.Copy(w, f) + if err != nil { + t.Errorf("copying failed: %v", err) + w.WriteHeader(http.StatusInternalServerError) + } + default: + t.Errorf("unknown request path: %q", r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + } + })) + + // The CISA KEV catalog uses HTTP/2, so might as well use it here, too. + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + return srv +} + +func TestFetch(t *testing.T) { + t.Parallel() + ctx := zlog.Test(context.Background(), t) + srv := mockServer(t) + + type fetchTestcase struct { + Name string + Check func(*testing.T, io.ReadCloser, driver.Fingerprint, error) + Hint string + } + + tt := []fetchTestcase{ + { + Name: "Fetch OK", // Tests successful fetch and data processing + Check: func(t *testing.T, rc io.ReadCloser, fp driver.Fingerprint, err error) { + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + defer rc.Close() + if rc == nil { + t.Error("expected non-nil ReadCloser for initial fetch") + } + if fp == "" { + t.Error("expected non-empty fingerprint") + } + t.Logf("fingerprint: %s", fp) + + // Further check if data is correctly read and structured + data, err := io.ReadAll(rc) + if err != nil { + t.Errorf("failed to read enrichment data: %v", err) + } + t.Logf("enrichment: %s", string(data)) + + if len(data) == 0 { + t.Error("expected non-empty data") + } + }, + }, + { + Name: "Fetch Unmodified", + Hint: lastModified, + Check: func(t *testing.T, rc io.ReadCloser, fp driver.Fingerprint, err error) { + if !errors.Is(err, driver.Unchanged) { + t.Errorf("unexpected error (or lack thereof): %v", err) + return + } + + if rc != nil { + t.Error("expected nil ReadCloser for initial fetch") + } + + if fp == "" { + t.Error("expected non-empty fingerprint") + } + t.Logf("fingerprint: %s", fp) + }, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + e := &Enricher{} + ctx := zlog.Test(ctx, t) + configFunc := func(i interface{}) error { + cfg, ok := i.(*Config) + if !ok { + t.Fatal("expected Config type for i, but got a different type") + } + u := srv.URL + "/known_exploited_vulnerabilities.json" + cfg.Feed = &u + return nil + } + + // Configure Enricher with mock server client and custom config + if err := e.Configure(ctx, configFunc, srv.Client()); err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + // Run FetchEnrichment and validate the result using Check + rc, fp, err := e.FetchEnrichment(ctx, driver.Fingerprint(tc.Hint)) + if rc != nil { + defer rc.Close() + } + if tc.Check != nil { + tc.Check(t, rc, fp, err) + } else if err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestParse(t *testing.T) { + t.Parallel() + ctx := zlog.Test(context.Background(), t) + srv := mockServer(t) + + e := &Enricher{} + f := func(i interface{}) error { + cfg, ok := i.(*Config) + if !ok { + t.Fatal("assertion failed") + } + u := srv.URL + "/known_exploited_vulnerabilities.json" + cfg.Feed = &u + return nil + } + if err := e.Configure(ctx, f, srv.Client()); err != nil { + t.Errorf("unexpected error: %v", err) + } + + rc, _, err := e.FetchEnrichment(ctx, "") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer rc.Close() + + rs, err := e.ParseEnrichment(ctx, rc) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + got := make(map[string]Entry) + for _, r := range rs { + if len(r.Tags) != 1 { + t.Errorf("unexpected number of tags: %d", len(r.Tags)) + } + var entry Entry + if err := json.Unmarshal(r.Enrichment, &entry); err != nil { + t.Errorf("unexpected error: %v", err) + } + got[r.Tags[0]] = entry + } + + want := map[string]Entry{ + "CVE-2017-3066": { + CVE: "CVE-2017-3066", + VulnerabilityName: "Adobe ColdFusion Deserialization Vulnerability", + CatalogVersion: "2025.02.24", + DateAdded: "2025-02-24", + ShortDescription: "Adobe ColdFusion contains a deserialization vulnerability in the Apache BlazeDS library that allows for arbitrary code execution.", + RequiredAction: "Apply mitigations per vendor instructions or discontinue use of the product if mitigations are unavailable.", + DueDate: "2025-03-17", + KnownRansomwareCampaignUse: "Unknown", + }, + "CVE-2021-44228": { + CVE: "CVE-2021-44228", + VulnerabilityName: "Apache Log4j2 Remote Code Execution Vulnerability", + CatalogVersion: "2025.02.24", + DateAdded: "2021-12-10", + ShortDescription: "Apache Log4j2 contains a vulnerability where JNDI features do not protect against attacker-controlled JNDI-related endpoints, allowing for remote code execution.", + RequiredAction: "For all affected software assets for which updates exist, the only acceptable remediation actions are: 1) Apply updates; OR 2) remove affected assets from agency networks. Temporary mitigations using one of the measures provided at https://www.cisa.gov/uscert/ed-22-02-apache-log4j-recommended-mitigation-measures are only acceptable until updates are available.", + DueDate: "2021-12-24", + KnownRansomwareCampaignUse: "Known", + }, + } + + if !cmp.Equal(got, want) { + t.Errorf("unexpected result, diff = %v", cmp.Diff(got, want)) + } +} + +type fakeGetter struct { + items []driver.EnrichmentRecord +} + +func (g fakeGetter) GetEnrichment(_ context.Context, cves []string) ([]driver.EnrichmentRecord, error) { + var results []driver.EnrichmentRecord + for _, cve := range cves { + for _, item := range g.items { + for _, tag := range item.Tags { + if tag == cve { + results = append(results, item) + break + } + } + } + } + return results, nil +} + +func TestEnrich(t *testing.T) { + t.Parallel() + ctx := zlog.Test(context.Background(), t) + srv := mockServer(t) + + e := &Enricher{} + f := func(i interface{}) error { + cfg, ok := i.(*Config) + if !ok { + t.Fatal("assertion failed") + } + u := srv.URL + "/known_exploited_vulnerabilities.json" + cfg.Feed = &u + return nil + } + if err := e.Configure(ctx, f, srv.Client()); err != nil { + t.Errorf("unexpected error: %v", err) + } + rc, _, err := e.FetchEnrichment(ctx, "") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer rc.Close() + rs, err := e.ParseEnrichment(ctx, rc) + if err != nil { + t.Fatal(err) + } + + g := &fakeGetter{items: rs} + r := &claircore.VulnerabilityReport{ + Vulnerabilities: map[string]*claircore.Vulnerability{ + "-1": { + Description: "This is a fake vulnerability that doesn't have a CVE.", + }, + "6004": { + Description: "CVE-2021-44228 is here", + }, + "6005": { + Description: "CVE-2017-3066 is awesome", + Links: "CVE-2017-3066.com", + }, + }, + } + kind, es, err := e.Enrich(ctx, g, r) + if err != nil { + t.Error(err) + } + if got, want := kind, Type; got != want { + t.Errorf("got: %q, want: %q", got, want) + } + want := map[string][]map[string]string{ + "6004": { + { + "cve": "CVE-2021-44228", + "vulnerability_name": "Apache Log4j2 Remote Code Execution Vulnerability", + "catalog_version": "2025.02.24", + "date_added": "2021-12-10", + "short_description": "Apache Log4j2 contains a vulnerability where JNDI features do not protect against attacker-controlled JNDI-related endpoints, allowing for remote code execution.", + "required_action": "For all affected software assets for which updates exist, the only acceptable remediation actions are: 1) Apply updates; OR 2) remove affected assets from agency networks. Temporary mitigations using one of the measures provided at https://www.cisa.gov/uscert/ed-22-02-apache-log4j-recommended-mitigation-measures are only acceptable until updates are available.", + "due_date": "2021-12-24", + "known_ransomware_campaign_use": "Known", + }, + }, + "6005": { + { + "cve": "CVE-2017-3066", + "vulnerability_name": "Adobe ColdFusion Deserialization Vulnerability", + "catalog_version": "2025.02.24", + "date_added": "2025-02-24", + "short_description": "Adobe ColdFusion contains a deserialization vulnerability in the Apache BlazeDS library that allows for arbitrary code execution.", + "required_action": "Apply mitigations per vendor instructions or discontinue use of the product if mitigations are unavailable.", + "due_date": "2025-03-17", + "known_ransomware_campaign_use": "Unknown", + }, + }, + } + + got := map[string][]map[string]string{} + if err := json.Unmarshal(es[0], &got); err != nil { + t.Error(err) + return + } + + log.Printf("Got: %+v\n", got) + if !cmp.Equal(got, want) { + t.Error(cmp.Diff(got, want)) + } +} diff --git a/enricher/kev/testdata/known_exploited_vulnerabilities.json b/enricher/kev/testdata/known_exploited_vulnerabilities.json new file mode 100644 index 000000000..6dd6a542b --- /dev/null +++ b/enricher/kev/testdata/known_exploited_vulnerabilities.json @@ -0,0 +1,40 @@ +{ + "title": "CISA Catalog of Known Exploited Vulnerabilities", + "catalogVersion": "2025.02.24", + "dateReleased": "2025-02-24T17:55:31.6365Z", + "count": 2, + "vulnerabilities": [ + { + "cveID": "CVE-2017-3066", + "vendorProject": "Adobe", + "product": "ColdFusion", + "vulnerabilityName": "Adobe ColdFusion Deserialization Vulnerability", + "dateAdded": "2025-02-24", + "shortDescription": "Adobe ColdFusion contains a deserialization vulnerability in the Apache BlazeDS library that allows for arbitrary code execution.", + "requiredAction": "Apply mitigations per vendor instructions or discontinue use of the product if mitigations are unavailable.", + "dueDate": "2025-03-17", + "knownRansomwareCampaignUse": "Unknown", + "notes": "https:\/\/helpx.adobe.com\/security\/products\/coldfusion\/apsb17-14.html ; https:\/\/nvd.nist.gov\/vuln\/detail\/CVE-2017-3066", + "cwes": [ + "CWE-502" + ] + }, + { + "cveID": "CVE-2021-44228", + "vendorProject": "Apache", + "product": "Log4j2", + "vulnerabilityName": "Apache Log4j2 Remote Code Execution Vulnerability", + "dateAdded": "2021-12-10", + "shortDescription": "Apache Log4j2 contains a vulnerability where JNDI features do not protect against attacker-controlled JNDI-related endpoints, allowing for remote code execution.", + "requiredAction": "For all affected software assets for which updates exist, the only acceptable remediation actions are: 1) Apply updates; OR 2) remove affected assets from agency networks. Temporary mitigations using one of the measures provided at https:\/\/www.cisa.gov\/uscert\/ed-22-02-apache-log4j-recommended-mitigation-measures are only acceptable until updates are available.", + "dueDate": "2021-12-24", + "knownRansomwareCampaignUse": "Known", + "notes": "https:\/\/nvd.nist.gov\/vuln\/detail\/CVE-2021-44228", + "cwes": [ + "CWE-20", + "CWE-400", + "CWE-502" + ] + } + ] +} diff --git a/enricher/kev/types.go b/enricher/kev/types.go new file mode 100644 index 000000000..4f3c1852f --- /dev/null +++ b/enricher/kev/types.go @@ -0,0 +1,34 @@ +package kev + +type Entry struct { + CVE string `json:"cve"` + VulnerabilityName string `json:"vulnerability_name"` + CatalogVersion string `json:"catalog_version"` + DateAdded string `json:"date_added"` + ShortDescription string `json:"short_description"` + RequiredAction string `json:"required_action"` + DueDate string `json:"due_date"` + KnownRansomwareCampaignUse string `json:"known_ransomware_campaign_use"` +} + +type Root struct { + Title string `json:"title,omitempty"` + CatalogVersion string `json:"catalogVersion"` + DateReleased string `json:"dateReleased"` + Count int `json:"count"` + Vulnerabilities []*Vulnerability `json:"vulnerabilities"` +} + +type Vulnerability struct { + CVEID string `json:"cveID"` + VendorProject string `json:"vendorProject"` + Product string `json:"product"` + VulnerabilityName string `json:"vulnerabilityName"` + DateAdded string `json:"dateAdded"` + ShortDescription string `json:"shortDescription"` + RequiredAction string `json:"requiredAction"` + DueDate string `json:"dueDate"` + KnownRansomwareCampaignUse string `json:"knownRansomwareCampaignUse,omitempty"` + Notes string `json:"notes,omitempty"` + CWEs []string `json:"cwes,omitempty"` +} From da5e661f057a6cbfb54cc5ba3cbbb24a5a4be544 Mon Sep 17 00:00:00 2001 From: RTann Date: Mon, 24 Feb 2025 16:31:33 -0800 Subject: [PATCH 3/6] default Signed-off-by: RTann --- libindex/libindex.go | 2 ++ matchers/defaults/defaults.go | 3 +++ updater/defaults/defaults.go | 6 ++++++ 3 files changed, 11 insertions(+) diff --git a/libindex/libindex.go b/libindex/libindex.go index b9e8874a1..45a7e4696 100644 --- a/libindex/libindex.go +++ b/libindex/libindex.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/quay/claircore/chainguard" "io" "net/http" "sort" @@ -96,6 +97,7 @@ func New(ctx context.Context, opts *Options, cl *http.Client) (*Libindex, error) opts.Ecosystems = []*indexer.Ecosystem{ dpkg.NewEcosystem(ctx), alpine.NewEcosystem(ctx), + chainguard.NewEcosystem(ctx), rhel.NewEcosystem(ctx), rpm.NewEcosystem(ctx), python.NewEcosystem(ctx), diff --git a/matchers/defaults/defaults.go b/matchers/defaults/defaults.go index 774e068f6..7201188d6 100644 --- a/matchers/defaults/defaults.go +++ b/matchers/defaults/defaults.go @@ -3,6 +3,7 @@ package defaults import ( "context" + "github.com/quay/claircore/chainguard" "sync" "time" @@ -46,6 +47,8 @@ func Error() error { var defaultMatchers = []driver.Matcher{ &alpine.Matcher{}, &aws.Matcher{}, + chainguard.ChainguardMatcher, + chainguard.WolfiMatcher, &debian.Matcher{}, &gobin.Matcher{}, &java.Matcher{}, diff --git a/updater/defaults/defaults.go b/updater/defaults/defaults.go index f3abbcbb8..da5f8522a 100644 --- a/updater/defaults/defaults.go +++ b/updater/defaults/defaults.go @@ -5,6 +5,7 @@ package defaults import ( "context" + "github.com/quay/claircore/chainguard" "sync" "time" @@ -55,6 +56,11 @@ func inner(ctx context.Context) error { return err } updater.Register("debian", df) + cf, err := chainguard.NewFactory(ctx) + if err != nil { + return err + } + updater.Register("chainguard", cf) updater.Register("osv", new(osv.Factory)) updater.Register("rhel-vex", new(vex.Factory)) From ac048f21f742141c28c935c3eee25c1998461701 Mon Sep 17 00:00:00 2001 From: RTann Date: Mon, 24 Feb 2025 16:47:40 -0800 Subject: [PATCH 4/6] factory Signed-off-by: RTann --- enricher/kev/kev.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/enricher/kev/kev.go b/enricher/kev/kev.go index 65e12d9df..1337ad1c9 100644 --- a/enricher/kev/kev.go +++ b/enricher/kev/kev.go @@ -47,6 +47,13 @@ func init() { } } +// NewFactory creates a Factory for the CISA KEV enricher. +func NewFactory() driver.UpdaterSetFactory { + set := driver.NewUpdaterSet() + _ = set.Add(&Enricher{}) + return driver.StaticSet(set) +} + // Enricher provides exploit data as enrichments to a VulnerabilityReport. // // Configure must be called before any other methods. From d7546ee6125ea869539572e571463893005a5f65 Mon Sep 17 00:00:00 2001 From: RTann Date: Wed, 26 Feb 2025 17:45:04 -0800 Subject: [PATCH 5/6] alma rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED Signed-off-by: RTann --- alma/distributionscanner.go | 89 ++++++++++++ alma/matcher.go | 50 +++++++ alma/parser.go | 94 +++++++++++++ alma/release.go | 30 ++++ alma/updater.go | 65 +++++++++ alma/updaterset.go | 132 ++++++++++++++++++ aws/client.go | 8 +- aws/updater.go | 12 +- aws/updater_test.go | 11 +- matchers/defaults/defaults.go | 2 + pkg/ovalutil/alma.go | 25 ++++ rpm/ecosystem.go | 2 + updater/defaults/defaults.go | 8 +- .../alas => updater/repomd}/repomd.go | 2 +- .../alas => updater/repomd}/repomd_test.go | 2 +- .../repomd}/testdata/test_repomd.xml | 0 .../repomd}/testdata/test_updateinfo.xml | 0 .../alas => updater/repomd}/updates.go | 2 +- .../alas => updater/repomd}/updates_test.go | 2 +- 19 files changed, 515 insertions(+), 21 deletions(-) create mode 100644 alma/distributionscanner.go create mode 100644 alma/matcher.go create mode 100644 alma/parser.go create mode 100644 alma/release.go create mode 100644 alma/updater.go create mode 100644 alma/updaterset.go create mode 100644 pkg/ovalutil/alma.go rename {aws/internal/alas => updater/repomd}/repomd.go (99%) rename {aws/internal/alas => updater/repomd}/repomd_test.go (99%) rename {aws/internal/alas => updater/repomd}/testdata/test_repomd.xml (100%) rename {aws/internal/alas => updater/repomd}/testdata/test_updateinfo.xml (100%) rename {aws/internal/alas => updater/repomd}/updates.go (99%) rename {aws/internal/alas => updater/repomd}/updates_test.go (98%) diff --git a/alma/distributionscanner.go b/alma/distributionscanner.go new file mode 100644 index 000000000..36d30e8df --- /dev/null +++ b/alma/distributionscanner.go @@ -0,0 +1,89 @@ +package alma + +import ( + "context" + "errors" + "fmt" + "io/fs" + "regexp" + "runtime/trace" + "strconv" + + "github.com/quay/zlog" + + "github.com/quay/claircore" + "github.com/quay/claircore/indexer" +) + +const ( + scannerName = "alma" + scannerVersion = "1" + scannerKind = "distribution" +) + +var cpeRegexp = regexp.MustCompile(`CPE_NAME="cpe:/o:almalinux:almalinux:(\d+)::baseos"`) + +var ( + _ indexer.DistributionScanner = (*DistributionScanner)(nil) + _ indexer.VersionedScanner = (*DistributionScanner)(nil) +) + +// DistributionScanner attempts to discover if a layer +// displays characteristics of an alma distribution +type DistributionScanner struct{} + +// Name implements scanner.VersionedScanner. +func (*DistributionScanner) Name() string { return scannerName } + +// Version implements scanner.VersionedScanner. +func (*DistributionScanner) Version() string { return scannerVersion } + +// Kind implements scanner.VersionedScanner. +func (*DistributionScanner) Kind() string { return scannerKind } + +func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) { + defer trace.StartRegion(ctx, "Scanner.Scan").End() + ctx = zlog.ContextWithValues(ctx, + "component", "alma/DistributionScanner.Scan", + "version", ds.Version(), + "layer", l.Hash.String()) + zlog.Debug(ctx).Msg("start") + defer zlog.Debug(ctx).Msg("done") + sys, err := l.FS() + if err != nil { + return nil, fmt.Errorf("alma: unable to open layer: %w", err) + } + d, err := findDistribution(sys) + if err != nil { + return nil, fmt.Errorf("alma: unexpected error reading files: %w", err) + } + if d == nil { + zlog.Debug(ctx).Msg("didn't find etc/os-release") + return nil, nil + } + return []*claircore.Distribution{d}, nil +} + +func findDistribution(sys fs.FS) (*claircore.Distribution, error) { + const osReleasePath = `etc/os-release` + b, err := fs.ReadFile(sys, osReleasePath) + switch { + case errors.Is(err, nil): + case errors.Is(err, fs.ErrNotExist): + return nil, nil + default: + return nil, fmt.Errorf("alma: unexpected error reading os-release file: %w", err) + } + ms := cpeRegexp.FindSubmatch(b) + if ms == nil { + return nil, nil + } + if len(ms) != 2 { + return nil, fmt.Errorf("alma: malformed os-release file: %q", b) + } + _, err = strconv.Atoi(string(ms[1])) + if err != nil { + return nil, fmt.Errorf("alma: unexpected error reading os-releasefile: %w", err) + } + return mkRelease(string(ms[1])), nil +} diff --git a/alma/matcher.go b/alma/matcher.go new file mode 100644 index 000000000..45b18bd06 --- /dev/null +++ b/alma/matcher.go @@ -0,0 +1,50 @@ +package alma + +import ( + "context" + + version "github.com/knqyf263/go-rpm-version" + + "github.com/quay/claircore" + "github.com/quay/claircore/libvuln/driver" +) + +type Matcher struct{} + +var _ driver.Matcher = (*Matcher)(nil) + +func (*Matcher) Name() string { + return "alma-matcher" +} + +func (*Matcher) Filter(record *claircore.IndexRecord) bool { + return record.Distribution != nil && record.Distribution.DID == "alma" +} + +// Query implements driver.Matcher +func (*Matcher) Query() []driver.MatchConstraint { + return []driver.MatchConstraint{ + driver.DistributionDID, + driver.DistributionName, + driver.DistributionVersion, + } +} + +func (*Matcher) Vulnerable(_ context.Context, record *claircore.IndexRecord, vuln *claircore.Vulnerability) (bool, error) { + pkgVer := version.NewVersion(record.Package.Version) + var vulnVer version.Version + // Assume the vulnerability record we have is for the last known vulnerable + // version, so greater versions aren't vulnerable. + cmp := func(i int) bool { return i != version.GREATER } + // But if it's explicitly marked as a fixed-in version, it's only vulnerable + // if less than that version. + if vuln.FixedInVersion != "" { + vulnVer = version.NewVersion(vuln.FixedInVersion) + cmp = func(i int) bool { return i == version.LESS } + } else { + // If a vulnerability doesn't have FixedInVersion, assume it is unfixed. + vulnVer = version.NewVersion("65535:0") + } + // compare version and architecture + return cmp(pkgVer.Compare(vulnVer)) && vuln.ArchOperation.Cmp(record.Package.Arch, vuln.Package.Arch), nil +} diff --git a/alma/parser.go b/alma/parser.go new file mode 100644 index 000000000..21bccdcfc --- /dev/null +++ b/alma/parser.go @@ -0,0 +1,94 @@ +package alma + +import ( + "context" + "encoding/xml" + "fmt" + "io" + "strings" + + "github.com/quay/goval-parser/oval" + "github.com/quay/zlog" + + "github.com/quay/claircore" + "github.com/quay/claircore/internal/xmlutil" + "github.com/quay/claircore/pkg/ovalutil" +) + +// Parse implements [driver.Updater]. +// +// Parse treats the data inside the provided io.ReadCloser as Red Hat +// flavored OVAL XML. The distribution associated with vulnerabilities +// is configured via the Updater. The repository associated with +// vulnerabilies is based on the affected CPE list. +func (u *Updater) Parse(ctx context.Context, r io.ReadCloser) ([]*claircore.Vulnerability, error) { + ctx = zlog.ContextWithValues(ctx, "component", "alma/Updater.Parse") + zlog.Info(ctx).Msg("starting parse") + defer r.Close() + root := oval.Root{} + dec := xml.NewDecoder(r) + dec.CharsetReader = xmlutil.CharsetReader + if err := dec.Decode(&root); err != nil { + return nil, fmt.Errorf("alma: unable to decode OVAL document: %w", err) + } + zlog.Debug(ctx).Msg("xml decoded") + protoVulns := func(def oval.Definition) ([]*claircore.Vulnerability, error) { + defType, err := ovalutil.GetAlmaDefinitionType(def) + if err != nil { + return nil, err + } + // Red Hat OVAL data include information about vulnerabilities, + // that actually don't affect the package in any way. Storing them + // would increase number of records in DB without adding any value. + if isSkippableDefinitionType(defType) { + return []*claircore.Vulnerability{}, nil + } + + // Go look for the vuln name in the references, fallback to + // title if not found. + name := def.Title + if len(def.References) > 0 { + name = def.References[0].RefID + } + + v := &claircore.Vulnerability{ + Updater: u.Name(), + Name: name, + Description: def.Description, + Issued: def.Advisory.Issued.Date, + Links: ovalutil.Links(def), + Severity: def.Advisory.Severity, + NormalizedSeverity: NormalizeSeverity(def.Advisory.Severity), + Dist: u.dist, + } + return []*claircore.Vulnerability{v}, nil + } + vulns, err := ovalutil.RPMDefsToVulns(ctx, &root, protoVulns) + if err != nil { + return nil, err + } + return vulns, nil +} + +func isSkippableDefinitionType(defType ovalutil.DefinitionType) bool { + return defType == ovalutil.UnaffectedDefinition || defType == ovalutil.NoneDefinition +} + +// NormalizeSeverity maps Red Hat severity strings to claircore's normalized +// serverity levels. +func NormalizeSeverity(severity string) claircore.Severity { + switch strings.ToLower(severity) { + case "none": + return claircore.Unknown + case "low": + return claircore.Low + case "moderate": + return claircore.Medium + case "important": + return claircore.High + case "critical": + return claircore.Critical + default: + return claircore.Unknown + } +} diff --git a/alma/release.go b/alma/release.go new file mode 100644 index 000000000..5508cf370 --- /dev/null +++ b/alma/release.go @@ -0,0 +1,30 @@ +package alma + +import ( + "sync" + + "github.com/quay/claircore" + "github.com/quay/claircore/toolkit/types/cpe" +) + +// RelMap memoizes the Distributions handed out by this package. +// +// Doing this is a cop-out to the previous approach of having a hard-coded set of structs. +// In the case something is (mistakenly) doing pointer comparisons, this will make that work +// but still allows us to have the list of distributions grow ad-hoc. +var relMap sync.Map + +func mkRelease(r string) *claircore.Distribution { + v, ok := relMap.Load(r) + if !ok { + v, _ = relMap.LoadOrStore(r, &claircore.Distribution{ + Name: "AlmaLinux", + Version: r, + VersionID: r, + DID: "alma", + PrettyName: "AlmaLinux " + r, + CPE: cpe.MustUnbind("cpe:/o:almalinux:almalinux:" + r), + }) + } + return v.(*claircore.Distribution) +} diff --git a/alma/updater.go b/alma/updater.go new file mode 100644 index 000000000..88956e54a --- /dev/null +++ b/alma/updater.go @@ -0,0 +1,65 @@ +package alma + +import ( + "context" + "fmt" + "github.com/quay/claircore" + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/pkg/ovalutil" + "github.com/quay/zlog" + "net/http" + "net/url" + "strconv" +) + +var ( + _ driver.Updater = (*Updater)(nil) + _ driver.Configurable = (*Updater)(nil) +) + +// Updater fetches and parses RHEL-flavored OVAL databases. +type Updater struct { + ovalutil.Fetcher // fetch method promoted via embed + dist *claircore.Distribution + name string +} + +// UpdaterConfig is the configuration expected for any given updater. +// +// See also [ovalutil.FetcherConfig]. +type UpdaterConfig struct { + ovalutil.FetcherConfig + Release int `json:"release" yaml:"release"` +} + +// NewUpdater returns an Updater. +func NewUpdater(release int, uri string) (*Updater, error) { + u := &Updater{ + name: fmt.Sprintf("alma-%d", release), + dist: mkRelease(strconv.Itoa(release)), + } + var err error + u.Fetcher.URL, err = url.Parse(uri) + if err != nil { + return nil, err + } + u.Fetcher.Compression = ovalutil.CompressionBzip2 + return u, nil +} + +// Configure implements [driver.Configurable]. +func (u *Updater) Configure(ctx context.Context, cf driver.ConfigUnmarshaler, c *http.Client) error { + ctx = zlog.ContextWithValues(ctx, "component", "rhel/Updater.Configure") + var cfg UpdaterConfig + if err := cf(&cfg); err != nil { + return err + } + if cfg.Release != 0 { + u.dist = mkRelease(strconv.Itoa(cfg.Release)) + } + + return u.Fetcher.Configure(ctx, cf, c) +} + +// Name implements [driver.Updater]. +func (u *Updater) Name() string { return u.name } diff --git a/alma/updaterset.go b/alma/updaterset.go new file mode 100644 index 000000000..18b7150aa --- /dev/null +++ b/alma/updaterset.go @@ -0,0 +1,132 @@ +package alma + +import ( + "context" + "fmt" + "github.com/quay/zlog" + "net/http" + "net/url" + + "github.com/quay/claircore/libvuln/driver" +) + +//doc:url updater +const dbURL = `https://security.almalinux.org/oval/` + +const ovalFmt = `org.almalinux.alsa-%d.xml.bz2` + +// NewFactory creates a Factory making updaters based on the contents of the +// provided pulp manifest. +func NewFactory(_ context.Context) (*Factory, error) { + return &Factory{}, nil +} + +// Factory contains the configuration for fetching and parsing a Pulp manifest. +type Factory struct { + base *url.URL + client *http.Client + etags map[int]string +} + +// FactoryConfig is the configuration accepted by the rhel updaters. +// +// By convention, this should be in a map called "rhel". +type FactoryConfig struct { + BaseURL string `json:"base_url" yaml:"base_url"` +} + +var _ driver.Configurable = (*Factory)(nil) + +// Configure implements [driver.Configurable]. +func (f *Factory) Configure(ctx context.Context, cfg driver.ConfigUnmarshaler, c *http.Client) error { + ctx = zlog.ContextWithValues(ctx, "component", "alma/Factory.Configure") + var fc FactoryConfig + + if err := cfg(&fc); err != nil { + return err + } + zlog.Debug(ctx).Msg("loaded incoming config") + + baseURL, err := url.Parse(dbURL) + if err != nil { + panic("programmer error: invalid Base URL") + } + f.base = baseURL + if fc.BaseURL != "" { + u, err := url.Parse(fc.BaseURL) + if err != nil { + return err + } + zlog.Info(ctx). + Stringer("base_url", u). + Msg("configured base URL") + f.base = u + } + + if c != nil { + zlog.Info(ctx). + Msg("configured HTTP client") + f.client = c + } + return nil +} + +// UpdaterSet implements [driver.UpdaterSetFactory]. +// +// The returned Updaters determine the [claircore.Distribution] it's associated +// with based on the path in the Pulp manifest. +func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { + ctx = zlog.ContextWithValues(ctx, "component", "alma/Factory.UpdaterSet") + + s := driver.NewUpdaterSet() + + etags := make(map[int]string) + + var done bool + for i := 8; !done; i++ { + u, err := f.base.Parse(fmt.Sprintf(ovalFmt, i)) + if err != nil { + return s, fmt.Errorf("alma: unable to construct request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil) + if err != nil { + return s, fmt.Errorf("alma: unable to construct request: %w", err) + } + if etag, exists := f.etags[i]; exists { + req.Header.Set("If-None-Match", etag) + } + + zlog.Debug(ctx).Msg("checking repository") + res, err := f.client.Do(req) + if err != nil { + return s, fmt.Errorf("alpine: error requesting %q: %w", u.String(), err) + } + _ = res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + zlog.Debug(ctx).Msg("found") + updater, err := NewUpdater(i, u.String()) + if err != nil { + return s, err + } + if err := s.Add(updater); err != nil { + return s, err + } + etags[i] = res.Header.Get("Etag") + case http.StatusNotModified: + zlog.Debug(ctx).Msg("not modified") + case http.StatusNotFound: + zlog.Debug(ctx).Msg("not found") + done = true + default: + zlog.Info(ctx).Str("status", res.Status).Msg("unexpected status reported") + } + } + + // Only add the etags if this is successful. + for k, v := range etags { + f.etags[k] = v + } + return s, nil +} diff --git a/aws/client.go b/aws/client.go index 8fe445e63..119286a22 100644 --- a/aws/client.go +++ b/aws/client.go @@ -6,6 +6,7 @@ import ( "encoding/xml" "errors" "fmt" + "github.com/quay/claircore/updater/repomd" "io" "net/http" "net/url" @@ -15,7 +16,6 @@ import ( "github.com/quay/zlog" - "github.com/quay/claircore/aws/internal/alas" "github.com/quay/claircore/internal/xmlutil" "github.com/quay/claircore/pkg/tmp" ) @@ -48,7 +48,7 @@ func NewClient(ctx context.Context, hc *http.Client, release Release) (*Client, } // RepoMD returns a alas.RepoMD containing sha256 information of a repositories contents -func (c *Client) RepoMD(ctx context.Context) (alas.RepoMD, error) { +func (c *Client) RepoMD(ctx context.Context) (repomd.RepoMD, error) { ctx = zlog.ContextWithValues(ctx, "component", "aws/Client.RepoMD") for _, mirror := range c.mirrors { m := *mirror @@ -80,7 +80,7 @@ func (c *Client) RepoMD(ctx context.Context) (alas.RepoMD, error) { continue } - repoMD := alas.RepoMD{} + repoMD := repomd.RepoMD{} dec := xml.NewDecoder(resp.Body) dec.CharsetReader = xmlutil.CharsetReader if err := dec.Decode(&repoMD); err != nil { @@ -95,7 +95,7 @@ func (c *Client) RepoMD(ctx context.Context) (alas.RepoMD, error) { } zlog.Error(ctx).Msg("exhausted all mirrors") - return alas.RepoMD{}, fmt.Errorf("all mirrors failed to retrieve repo metadata") + return repomd.RepoMD{}, fmt.Errorf("all mirrors failed to retrieve repo metadata") } // Updates returns the *http.Response of the first mirror to establish a connection diff --git a/aws/updater.go b/aws/updater.go index 16a550ab1..9efd1b489 100644 --- a/aws/updater.go +++ b/aws/updater.go @@ -4,6 +4,7 @@ import ( "context" "encoding/xml" "fmt" + "github.com/quay/claircore/updater/repomd" "io" "net/http" "strings" @@ -12,7 +13,6 @@ import ( "github.com/quay/zlog" "github.com/quay/claircore" - "github.com/quay/claircore/aws/internal/alas" "github.com/quay/claircore/internal/xmlutil" "github.com/quay/claircore/libvuln/driver" ) @@ -56,7 +56,7 @@ func (u *Updater) Fetch(ctx context.Context, fingerprint driver.Fingerprint) (io return nil, "", fmt.Errorf("failed to retrieve repo metadata: %v", err) } - updatesRepoMD, err := repoMD.Repo(alas.UpdateInfo, "") + updatesRepoMD, err := repoMD.Repo(repomd.UpdateInfo, "") if err != nil { return nil, "", fmt.Errorf("updates repo metadata could not be retrieved: %v", err) } @@ -75,7 +75,7 @@ func (u *Updater) Fetch(ctx context.Context, fingerprint driver.Fingerprint) (io } func (u *Updater) Parse(ctx context.Context, contents io.ReadCloser) ([]*claircore.Vulnerability, error) { - var updates alas.Updates + var updates repomd.Updates dec := xml.NewDecoder(contents) dec.CharsetReader = xmlutil.CharsetReader if err := dec.Decode(&updates); err != nil { @@ -104,7 +104,7 @@ func (u *Updater) Parse(ctx context.Context, contents io.ReadCloser) ([]*clairco // unpack takes the partially populated vulnerability and creates a fully populated vulnerability for each // provided alas.Package -func (u *Updater) unpack(partial *claircore.Vulnerability, packages []alas.Package) []*claircore.Vulnerability { +func (u *Updater) unpack(partial *claircore.Vulnerability, packages []repomd.Package) []*claircore.Vulnerability { out := []*claircore.Vulnerability{} var b strings.Builder @@ -126,7 +126,7 @@ func (u *Updater) unpack(partial *claircore.Vulnerability, packages []alas.Packa return out } -func versionString(b *strings.Builder, p alas.Package) string { +func versionString(b *strings.Builder, p repomd.Package) string { b.Reset() if p.Epoch != "" && p.Epoch != "0" { @@ -141,7 +141,7 @@ func versionString(b *strings.Builder, p alas.Package) string { } // refsToLinks takes an alas.Update and creates a string with all the href links -func refsToLinks(u alas.Update) string { +func refsToLinks(u repomd.Update) string { out := []string{} for _, ref := range u.References { out = append(out, ref.Href) diff --git a/aws/updater_test.go b/aws/updater_test.go index a8b669dd7..d543bb5fa 100644 --- a/aws/updater_test.go +++ b/aws/updater_test.go @@ -1,21 +1,20 @@ package aws import ( + "github.com/quay/claircore/updater/repomd" "strings" "testing" "github.com/google/go-cmp/cmp" - - "github.com/quay/claircore/aws/internal/alas" ) func TestVersionString(t *testing.T) { testcases := []struct { - pkg alas.Package + pkg repomd.Package expected string }{ { - pkg: alas.Package{ + pkg: repomd.Package{ Epoch: "", Version: "3.3.10", Release: "26.amzn2", @@ -23,7 +22,7 @@ func TestVersionString(t *testing.T) { expected: "3.3.10-26.amzn2", }, { - pkg: alas.Package{ + pkg: repomd.Package{ Epoch: "0", Version: "3.3.10", Release: "26.amzn2", @@ -31,7 +30,7 @@ func TestVersionString(t *testing.T) { expected: "3.3.10-26.amzn2", }, { - pkg: alas.Package{ + pkg: repomd.Package{ Epoch: "10", Version: "3.1.0", Release: "8.amzn2.0.8", diff --git a/matchers/defaults/defaults.go b/matchers/defaults/defaults.go index 7201188d6..3fc5c2630 100644 --- a/matchers/defaults/defaults.go +++ b/matchers/defaults/defaults.go @@ -3,6 +3,7 @@ package defaults import ( "context" + "github.com/quay/claircore/alma" "github.com/quay/claircore/chainguard" "sync" "time" @@ -45,6 +46,7 @@ func Error() error { // all the matchers libvuln will use to match // index records to vulnerabilities. var defaultMatchers = []driver.Matcher{ + &alma.Matcher{}, &alpine.Matcher{}, &aws.Matcher{}, chainguard.ChainguardMatcher, diff --git a/pkg/ovalutil/alma.go b/pkg/ovalutil/alma.go new file mode 100644 index 000000000..2941e868e --- /dev/null +++ b/pkg/ovalutil/alma.go @@ -0,0 +1,25 @@ +package ovalutil + +import ( + "errors" + "regexp" + + "github.com/quay/goval-parser/oval" +) + +const ( + ALEADefinition DefinitionType = "alea" + ALBADefinition DefinitionType = "alba" + ALSADefinition DefinitionType = "alsa" +) + +var almaDefinitionTypeRegex = regexp.MustCompile(`^oval:org\.almalinux\.([a-z]+):def:\d+$`) + +// GetAlmaDefinitionType parses an OVAL definition and extracts its type from ID. +func GetAlmaDefinitionType(def oval.Definition) (DefinitionType, error) { + match := almaDefinitionTypeRegex.FindStringSubmatch(def.ID) + if len(match) != 2 { // we should have match of the whole string and one submatch + return "", errors.New("cannot parse definition ID for its type") + } + return DefinitionType(match[1]), nil +} diff --git a/rpm/ecosystem.go b/rpm/ecosystem.go index 9ee224b5f..6474ff28d 100644 --- a/rpm/ecosystem.go +++ b/rpm/ecosystem.go @@ -2,6 +2,7 @@ package rpm import ( "context" + "github.com/quay/claircore/alma" "github.com/quay/claircore/aws" "github.com/quay/claircore/indexer" @@ -19,6 +20,7 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem { }, DistributionScanners: func(ctx context.Context) ([]indexer.DistributionScanner, error) { return []indexer.DistributionScanner{ + &alma.DistributionScanner{}, &aws.DistributionScanner{}, &oracle.DistributionScanner{}, &suse.DistributionScanner{}, diff --git a/updater/defaults/defaults.go b/updater/defaults/defaults.go index da5f8522a..a9966f4b3 100644 --- a/updater/defaults/defaults.go +++ b/updater/defaults/defaults.go @@ -5,12 +5,13 @@ package defaults import ( "context" - "github.com/quay/claircore/chainguard" + "github.com/quay/claircore/alma" "sync" "time" "github.com/quay/claircore/alpine" "github.com/quay/claircore/aws" + "github.com/quay/claircore/chainguard" "github.com/quay/claircore/debian" "github.com/quay/claircore/enricher/cvss" "github.com/quay/claircore/libvuln/driver" @@ -41,6 +42,11 @@ func Error() error { } func inner(ctx context.Context) error { + almaF, err := alma.NewFactory(ctx) + if err != nil { + return err + } + updater.Register("alma", almaF) af, err := alpine.NewFactory(ctx) if err != nil { return err diff --git a/aws/internal/alas/repomd.go b/updater/repomd/repomd.go similarity index 99% rename from aws/internal/alas/repomd.go rename to updater/repomd/repomd.go index e0665b291..c174c56ad 100644 --- a/aws/internal/alas/repomd.go +++ b/updater/repomd/repomd.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alas +package repomd import ( "errors" diff --git a/aws/internal/alas/repomd_test.go b/updater/repomd/repomd_test.go similarity index 99% rename from aws/internal/alas/repomd_test.go rename to updater/repomd/repomd_test.go index af333a1bb..1ebae8ad5 100644 --- a/aws/internal/alas/repomd_test.go +++ b/updater/repomd/repomd_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alas +package repomd import ( "encoding/xml" diff --git a/aws/internal/alas/testdata/test_repomd.xml b/updater/repomd/testdata/test_repomd.xml similarity index 100% rename from aws/internal/alas/testdata/test_repomd.xml rename to updater/repomd/testdata/test_repomd.xml diff --git a/aws/internal/alas/testdata/test_updateinfo.xml b/updater/repomd/testdata/test_updateinfo.xml similarity index 100% rename from aws/internal/alas/testdata/test_updateinfo.xml rename to updater/repomd/testdata/test_updateinfo.xml diff --git a/aws/internal/alas/updates.go b/updater/repomd/updates.go similarity index 99% rename from aws/internal/alas/updates.go rename to updater/repomd/updates.go index 5c3e062e1..7e1c5cc48 100644 --- a/aws/internal/alas/updates.go +++ b/updater/repomd/updates.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alas +package repomd import ( "encoding/xml" diff --git a/aws/internal/alas/updates_test.go b/updater/repomd/updates_test.go similarity index 98% rename from aws/internal/alas/updates_test.go rename to updater/repomd/updates_test.go index 85b0f6dd4..ac25e5db6 100644 --- a/aws/internal/alas/updates_test.go +++ b/updater/repomd/updates_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alas +package repomd import ( "encoding/xml" From 4e82c070ca29c42402408b54d611f9b5ef9dd2b9 Mon Sep 17 00:00:00 2001 From: RTann Date: Thu, 27 Feb 2025 00:21:55 -0800 Subject: [PATCH 6/6] non-nil map rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- alma/updaterset.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alma/updaterset.go b/alma/updaterset.go index 18b7150aa..cf085d96c 100644 --- a/alma/updaterset.go +++ b/alma/updaterset.go @@ -6,6 +6,7 @@ import ( "github.com/quay/zlog" "net/http" "net/url" + "strconv" "github.com/quay/claircore/libvuln/driver" ) @@ -18,7 +19,7 @@ const ovalFmt = `org.almalinux.alsa-%d.xml.bz2` // NewFactory creates a Factory making updaters based on the contents of the // provided pulp manifest. func NewFactory(_ context.Context) (*Factory, error) { - return &Factory{}, nil + return &Factory{etags: make(map[int]string)}, nil } // Factory contains the configuration for fetching and parsing a Pulp manifest. @@ -84,6 +85,7 @@ func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { var done bool for i := 8; !done; i++ { + ctx = zlog.ContextWithValues(ctx, "version", strconv.Itoa(i)) u, err := f.base.Parse(fmt.Sprintf(ovalFmt, i)) if err != nil { return s, fmt.Errorf("alma: unable to construct request: %w", err)