moderate review comments

Signed-off-by: Alex Goodman <[email protected]>

DEVELOPING.md

@@ -7,7 +7,8 @@ well as acceptance testing. You will require the following:
 
 - Python 3.8+ installed on your system. Consider using [pyenv](https://github.com/pyenv/pyenv) if you do not have a 
   preference for managing python interpreter installations.
-
+- `zstd` binary utility if you are packaging v6+ DB schemas
+- _(optional)_ `xz` binary utility if you have specifically overridden the package command options
 
 - [Poetry](https://python-poetry.org/) installed for dependency and virtualenv management for python dependencies, to install:
 

README.md

@@ -21,6 +21,8 @@ curl -sSfL https://raw.githubusercontent.com/anchore/grype-db/main/install.sh |
 curl -sSfL https://raw.githubusercontent.com/anchore/grype-db/main/install.sh | sh -s -- -b <DESTINATION_DIR> <RELEASE_VERSION>
 ```
 
+> [!IMPORTANT]  
+> You will require the `zstd` utility installed on your system to support the `package` command.
 
 ## Usage
 
@@ -39,6 +41,7 @@ grype-db pull [-g] [-p PROVIDER ...]
 grype-db build [-g] [--dir=DIR] [--schema=SCHEMA] [--skip-validation] [-p PROVIDER ...]
 
 # Package the already built DB file into an archive ready for upload and serving
+# note: you will require the zstd utility to be installed on your system
 grype-db package [--dir=DIR] [--publish-base-url=URL]
 ```
 
@@ -54,7 +57,7 @@ is created that is used in packaging and curation of the database file by this a
 and a `provider-metadata.json` file is created that includes the last successful run date for each provider.
 Use `-g` to generate the list of providers to pull based on the output of "vunnel list".
 
-The `package` command archives the `vulnerability.db`, `metadata.json` and `provider-metadata.json` files into a `tar.gz` file. Additionally, a `listing.json`
+The `package` command archives the `vulnerability.db` file into a `tar.zstd` file. Additionally, a `latest.json`
 is generated to aid in serving one or more database archives for downstream consumption, where the consuming application should
 use the listing file to discover available archives available for download. The base URL used to create the download URL for each
 database archive is controlled by the `package.base-url` configuration option.

cmd/grype-db/cli/commands/build.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"time"
 
 	"github.com/scylladb/go-set/strset"
 	"github.com/spf13/cobra"
@@ -108,7 +107,7 @@ func runBuild(cfg buildConfig) error {
 		return fmt.Errorf("unable to get provider states: %w", err)
 	}
 
-	earliest, err := earliestTimestamp(states)
+	earliest, err := provider.States(states).EarliestTimestamp()
 	if err != nil {
 		return fmt.Errorf("unable to get earliest timestamp: %w", err)
 	}
@@ -152,31 +151,3 @@ func providerStates(skipValidation bool, providers []provider.Provider) ([]provi
 	}
 	return states, nil
 }
-
-func earliestTimestamp(states []provider.State) (time.Time, error) {
-	if len(states) == 0 {
-		return time.Time{}, fmt.Errorf("cannot find earliest timestamp: no states provided")
-	}
-	var earliest time.Time
-	for _, s := range states {
-		// the NVD api is constantly down, so we don't want to consider it for the earliest timestamp
-		if s.Provider == "nvd" {
-			log.WithFields("provider", s.Provider).Debug("not considering data age for provider")
-			continue
-		}
-		if earliest.IsZero() {
-			earliest = s.Timestamp
-			continue
-		}
-		if s.Timestamp.Before(earliest) {
-			earliest = s.Timestamp
-		}
-	}
-
-	if earliest.IsZero() {
-		return time.Time{}, fmt.Errorf("unable to determine earliest timestamp")
-	}
-
-	log.WithFields("timestamp", earliest).Debug("earliest data timestamp")
-	return earliest, nil
-}

internal/tarutil/writer.go

@@ -73,6 +73,15 @@ func newShellCompressor(c string, archive io.Writer) (*shellCompressor, error) {
 		return nil, fmt.Errorf("unable to parse command: %w", err)
 	}
 	binary := args[0]
+
+	binPath, err := exec.LookPath(binary)
+	if err != nil {
+		return nil, fmt.Errorf("unable to find binary %q: %w", binary, err)
+	}
+	if binPath == "" {
+		return nil, fmt.Errorf("unable to find binary %q in PATH", binary)
+	}
+
 	args = args[1:]
 	cmd := exec.Command(binary, args...)
 	log.Debug(strings.Join(cmd.Args, " "))

pkg/process/package.go

@@ -9,8 +9,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/anchore/grype-db/internal/log"
 	"github.com/anchore/grype-db/internal/tarutil"
+	"github.com/anchore/grype-db/pkg/provider"
 	grypeDBLegacyDistribution "github.com/anchore/grype/grype/db/legacy/distribution"
 	v6 "github.com/anchore/grype/grype/db/v6"
 	v6Distribution "github.com/anchore/grype/grype/db/v6/distribution"
@@ -31,69 +34,34 @@ func packageDB(dbDir, overrideArchiveExtension string) error {
 	}
 	log.WithFields("from", dbDir, "extension", extension).Info("packaging database")
 
-	tarPath, err := calculateTarPath(dbDir, extension)
-	if err != nil {
-		return err
-	}
-
-	if err := populateTar(tarPath); err != nil {
-		return err
-	}
-
-	log.WithFields("path", tarPath).Info("created database archive")
-
-	return writeLatestDocument(tarPath)
-}
-
-func resolveExtension(overrideArchiveExtension string) (string, error) {
-	var extension = "tar.zst"
-
-	if overrideArchiveExtension != "" {
-		extension = strings.TrimLeft(overrideArchiveExtension, ".")
-	}
-
-	var found bool
-	for _, valid := range []string{"tar.zst", "tar.xz", "tar.gz"} {
-		if valid == extension {
-			found = true
-			break
-		}
-	}
-
-	if !found {
-		return "", fmt.Errorf("unsupported archive extension %q", extension)
-	}
-	return extension, nil
-}
-
-func calculateTarPath(dbDir string, extension string) (string, error) {
 	s, err := v6.NewReader(v6.Config{DBDirPath: dbDir})
 	if err != nil {
-		return "", fmt.Errorf("unable to open vulnerability store: %w", err)
+		return fmt.Errorf("unable to open vulnerability store: %w", err)
 	}
 
 	metadata, err := s.GetDBMetadata()
-	if err != nil {
-		return "", fmt.Errorf("unable to get vulnerability store metadata: %w", err)
+	if err != nil || metadata == nil {
+		return fmt.Errorf("unable to get vulnerability store metadata: %w", err)
 	}
 
 	if metadata.Model != v6.ModelVersion {
-		return "", fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model)
+		return fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model)
 	}
 
-	providers, err := s.AllProviders()
+	providerModels, err := s.AllProviders()
 	if err != nil {
-		return "", fmt.Errorf("unable to get all providers: %w", err)
+		return fmt.Errorf("unable to get all providers: %w", err)
 	}
 
-	if len(providers) == 0 {
-		return "", fmt.Errorf("no providers found in the vulnerability store")
+	if len(providerModels) == 0 {
+		return fmt.Errorf("no providers found in the vulnerability store")
 	}
 
-	eldest := eldestProviderTimestamp(providers)
-	if eldest == nil {
-		return "", errors.New("could not resolve eldest provider timestamp")
+	eldest, err := toProviders(providerModels).EarliestTimestamp()
+	if err != nil {
+		return err
 	}
+
 	// output archive vulnerability-db_VERSION_OLDESTDATADATE_BUILTEPOCH.tar.gz, where:
 	// - VERSION: schema version in the form of v#.#.#
 	// - OLDESTDATADATE: RFC3338 formatted value of the oldest date capture date found for all contained providers
@@ -106,19 +74,51 @@ func calculateTarPath(dbDir string, extension string) (string, error) {
 		extension,
 	)
 
-	return filepath.Join(dbDir, tarName), err
+	tarPath := filepath.Join(dbDir, tarName)
+
+	if err := populateTar(tarPath); err != nil {
+		return err
+	}
+
+	log.WithFields("path", tarPath).Info("created database archive")
+
+	return writeLatestDocument(tarPath, *metadata)
+}
+
+func toProviders(states []v6.Provider) provider.States {
+	var result provider.States
+	for _, state := range states {
+		result = append(result, provider.State{
+			Provider:  state.ID,
+			Timestamp: *state.DateCaptured,
+		})
+	}
+	return result
 }
 
-func eldestProviderTimestamp(providers []v6.Provider) *time.Time {
-	var eldest *time.Time
-	for _, p := range providers {
-		if eldest == nil || p.DateCaptured.Before(*eldest) {
-			eldest = p.DateCaptured
+func resolveExtension(overrideArchiveExtension string) (string, error) {
+	var extension = "tar.zst"
+
+	if overrideArchiveExtension != "" {
+		extension = strings.TrimLeft(overrideArchiveExtension, ".")
+	}
+
+	var found bool
+	for _, valid := range []string{"tar.zst", "tar.xz", "tar.gz"} {
+		if valid == extension {
+			found = true
+			break
 		}
 	}
-	return eldest
+
+	if !found {
+		return "", fmt.Errorf("unsupported archive extension %q", extension)
+	}
+	return extension, nil
 }
 
+var listingFiles = strset.New("listing.json", "latest.json", "history.json")
+
 func populateTar(tarPath string) error {
 	originalDir, err := os.Getwd()
 	if err != nil {
@@ -146,7 +146,7 @@ func populateTar(tarPath string) error {
 
 	var files []string
 	for _, fi := range fileInfos {
-		if fi.Name() != "listing.json" && !strings.Contains(fi.Name(), ".tar.") {
+		if !listingFiles.Has(fi.Name()) && !strings.Contains(fi.Name(), ".tar.") {
 			files = append(files, fi.Name())
 		}
 	}
@@ -158,8 +158,8 @@ func populateTar(tarPath string) error {
 	return nil
 }
 
-func writeLatestDocument(tarPath string) error {
-	archive, err := v6Distribution.NewArchive(tarPath)
+func writeLatestDocument(tarPath string, metadata v6.DBMetadata) error {
+	archive, err := v6Distribution.NewArchive(tarPath, *metadata.BuildTimestamp, metadata.Model, metadata.Revision, metadata.Addition)
 	if err != nil || archive == nil {
 		return fmt.Errorf("unable to create archive: %w", err)
 	}

pkg/provider/state.go

@@ -134,3 +134,31 @@ func (s States) Names() []string {
 	}
 	return names
 }
+
+func (s States) EarliestTimestamp() (time.Time, error) {
+	if len(s) == 0 {
+		return time.Time{}, fmt.Errorf("cannot find earliest timestamp: no states provided")
+	}
+	var earliest time.Time
+	for _, curState := range s {
+		// the NVD api is constantly down, so we don't want to consider it for the earliest timestamp
+		if curState.Provider == "nvd" {
+			log.WithFields("provider", curState.Provider).Debug("not considering data age for provider")
+			continue
+		}
+		if earliest.IsZero() {
+			earliest = curState.Timestamp
+			continue
+		}
+		if curState.Timestamp.Before(earliest) {
+			earliest = curState.Timestamp
+		}
+	}
+
+	if earliest.IsZero() {
+		return time.Time{}, fmt.Errorf("unable to determine earliest timestamp")
+	}
+
+	log.WithFields("timestamp", earliest).Debug("earliest data timestamp")
+	return earliest, nil
+}

cmd/grype-db/cli/commands/build_test.go → pkg/provider/state_test.go

@@ -1,25 +1,23 @@
-package commands
+package provider
 
 import (
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
-
-	"github.com/anchore/grype-db/pkg/provider"
 )
 
 func Test_earliestTimestamp(t *testing.T) {
 	tests := []struct {
 		name    string
-		states  []provider.State
+		states  []State
 		want    time.Time
 		wantErr require.ErrorAssertionFunc
 	}{
 		{
 			name: "happy path",
-			states: []provider.State{
+			states: []State{
 				{
 					Timestamp: time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC),
 				},
@@ -34,13 +32,13 @@ func Test_earliestTimestamp(t *testing.T) {
 		},
 		{
 			name:    "empty states",
-			states:  []provider.State{},
+			states:  []State{},
 			want:    time.Time{},
 			wantErr: requireErrorContains("cannot find earliest timestamp: no states provided"),
 		},
 		{
 			name: "single state",
-			states: []provider.State{
+			states: []State{
 				{
 					Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
 				},
@@ -49,7 +47,7 @@ func Test_earliestTimestamp(t *testing.T) {
 		},
 		{
 			name: "all states have provider nvd",
-			states: []provider.State{
+			states: []State{
 				{
 					Provider:  "nvd",
 					Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
@@ -64,7 +62,7 @@ func Test_earliestTimestamp(t *testing.T) {
 		},
 		{
 			name: "mix of nvd and non-nvd providers",
-			states: []provider.State{
+			states: []State{
 				{
 					Provider:  "nvd",
 					Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
@@ -82,7 +80,7 @@ func Test_earliestTimestamp(t *testing.T) {
 		},
 		{
 			name: "timestamps are the same",
-			states: []provider.State{
+			states: []State{
 				{
 					Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
 				},
@@ -102,7 +100,7 @@ func Test_earliestTimestamp(t *testing.T) {
 			if tt.wantErr == nil {
 				tt.wantErr = require.NoError
 			}
-			got, err := earliestTimestamp(tt.states)
+			got, err := States(tt.states).EarliestTimestamp()
 			tt.wantErr(t, err)
 			if err != nil {
 				return