Merge pull request #146 from amartin120/more-updates-0.4.1

Improved logging for store copy / Updated store info to handle multi-arch images

cmd/hauler/cli/store/copy.go

@@ -63,7 +63,7 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string
 			}
 		}
 
-		err := cosign.LoadImage(ctx, s, components[1], ropts)
+		err := cosign.LoadImages(ctx, s, components[1], ropts)
 		if err != nil {
 			return err
 		}
@@ -72,6 +72,6 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string
 		return fmt.Errorf("detecting protocol from [%s]", targetRef)
 	}
 
-	l.Infof("Copied artifacts to [%s]", components[1])
+	l.Infof("copied artifacts to [%s]", components[1])
 	return nil
 }

cmd/hauler/cli/store/info.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"strings"
-	"text/tabwriter"
+	"github.com/olekukonko/tablewriter"
+	"os"
+	"sort"
 
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
@@ -44,49 +45,109 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
 		}
 		defer rc.Close()
 
-		var m ocispec.Manifest
-		if err := json.NewDecoder(rc).Decode(&m); err != nil {
-			return err
-		}
-		i := newItem(s, desc, m)
-		var emptyItem item
-		if i != emptyItem {
-			items = append(items, i)
+		// handle multi-arch images 
+		if desc.MediaType == consts.OCIImageIndexSchema || desc.MediaType == consts.DockerManifestListSchema2 {
+			var idx ocispec.Index
+			if err := json.NewDecoder(rc).Decode(&idx); err != nil {
+				return err
+			}
+
+			for _, internalDesc := range idx.Manifests {
+				rc, err := s.Fetch(ctx, internalDesc)
+				if err != nil {
+					return err
+				}
+				defer rc.Close()
+
+				var internalManifest ocispec.Manifest
+				if err := json.NewDecoder(rc).Decode(&internalManifest); err != nil {
+					return err
+				}
+
+				i := newItem(s, desc, internalManifest, internalDesc.Platform.Architecture)
+				var emptyItem item
+				if i != emptyItem {
+					items = append(items, i)
+				}
+			}
+		// handle single arch docker images
+		} else if desc.MediaType == consts.DockerManifestSchema2 {
+			var m ocispec.Manifest
+			if err := json.NewDecoder(rc).Decode(&m); err != nil {
+				return err
+			}
+
+			rc, err := s.FetchManifest(ctx, m)
+			if err != nil {
+				return err
+			}
+			defer rc.Close()
+
+			// Unmarshal the OCI image content
+			var internalManifest ocispec.Image
+			if err := json.NewDecoder(rc).Decode(&internalManifest); err != nil {
+				return err
+			}
+
+			i := newItem(s, desc, m, internalManifest.Architecture)
+			var emptyItem item
+			if i != emptyItem {
+				items = append(items, i)
+			}
+		// handle the rest
+		} else {  
+			var m ocispec.Manifest
+			if err := json.NewDecoder(rc).Decode(&m); err != nil {
+				return err
+			}
+
+			i := newItem(s, desc, m, "-")
+			var emptyItem item
+			if i != emptyItem {
+				items = append(items, i)
+			}
 		}
 
 		return nil
 	}); err != nil {
 		return err
 	}
 
+	// sort items by ref and arch
+	sort.Sort(byReferenceAndArch(items))
+
 	var msg string
 	switch o.OutputFormat {
 	case "json":
 		msg = buildJson(items...)
-
+		fmt.Println(msg)
 	default:
-		msg = buildTable(items...)
+		buildTable(items...)
 	}
-	fmt.Println(msg)
 	return nil
 }
 
-func buildTable(items ...item) string {
-	b := strings.Builder{}
-	tw := tabwriter.NewWriter(&b, 1, 1, 3, ' ', 0)
-
-	fmt.Fprintf(tw, "Reference\tType\t# Layers\tSize\n")
-	fmt.Fprintf(tw, "---------\t----\t--------\t----\n")
+func buildTable(items ...item) {
+	// Create a table for the results
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Reference", "Type", "Arch", "# Layers", "Size"})
+	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+	table.SetRowLine(false)
+	table.SetAutoMergeCellsByColumnIndex([]int{0})
 
 	for _, i := range items {
 		if i.Type != "" {
-			fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n",
-				i.Reference, i.Type, i.Layers, i.Size,
-			)
+			row := []string{
+				i.Reference,
+				i.Type,
+				i.Architecture,
+				fmt.Sprintf("%d", i.Layers),
+				i.Size,
+			}
+			table.Append(row)
 		}
 	}
-	tw.Flush()
-	return b.String()
+	table.Render()
 }
 
 func buildJson(item ...item) string {
@@ -98,16 +159,29 @@ func buildJson(item ...item) string {
 }
 
 type item struct {
-	Reference string
-	Type      string
-	Layers    int
-	Size      string
+	Reference    string
+	Type         string
+	Architecture string
+	Layers       int
+	Size         string
+}
+
+type byReferenceAndArch []item
+
+func (a byReferenceAndArch) Len() int      { return len(a) }
+func (a byReferenceAndArch) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byReferenceAndArch) Less(i, j int) bool {
+	if a[i].Reference == a[j].Reference {
+		return a[i].Architecture < a[j].Architecture
+	}
+	return a[i].Reference < a[j].Reference
 }
 
-func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item {
+func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest, arch string) item {
+	// skip listing cosign items
 	if desc.Annotations["kind"] == "dev.cosignproject.cosign/atts" ||
-	   desc.Annotations["kind"] == "dev.cosignproject.cosign/sigs" ||
-	   desc.Annotations["kind"] == "dev.cosignproject.cosign/sboms" {
+		desc.Annotations["kind"] == "dev.cosignproject.cosign/sigs" ||
+		desc.Annotations["kind"] == "dev.cosignproject.cosign/sboms" {
 		return item{}
 	}
 
@@ -135,10 +209,11 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item
 	}
 
 	return item{
-		Reference: ref.Name(),
-		Type:      ctype,
-		Layers:    len(m.Layers),
-		Size:      byteCountSI(size),
+		Reference:    ref.Name(),
+		Type:         ctype,
+		Architecture: arch,
+		Layers:       len(m.Layers),
+		Size:         byteCountSI(size),
 	}
 }
 

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc5
 	github.com/pkg/errors v0.9.1

go.sum

@@ -417,6 +417,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
 github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=

pkg/consts/consts.go

@@ -4,6 +4,7 @@ const (
 	OCIManifestSchema1        = "application/vnd.oci.image.manifest.v1+json"
 	DockerManifestSchema2     = "application/vnd.docker.distribution.manifest.v2+json"
 	DockerManifestListSchema2 = "application/vnd.docker.distribution.manifest.list.v2+json"
+	OCIImageIndexSchema       = "application/vnd.oci.image.index.v1+json"
 
 	DockerConfigJSON        = "application/vnd.docker.container.image.v1+json"
 	DockerLayer             = "application/vnd.docker.image.rootfs.diff.tar.gzip"

pkg/content/oci.go

@@ -161,6 +161,14 @@ func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser
 	return readerAt, nil
 }
 
+func (o *OCI) FetchManifest(ctx context.Context, manifest ocispec.Manifest) (io.ReadCloser, error) {
+	readerAt, err := o.manifestBlobReaderAt(manifest)
+	if err != nil {
+		return nil, err
+	}
+	return readerAt, nil
+}
+
 // Pusher returns a new pusher for the provided reference
 // The returned Pusher should satisfy content.Ingester and concurrent attempts
 // to push the same blob using the Ingester API should result in ErrUnavailable.
@@ -208,6 +216,14 @@ func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) {
 	return os.Open(blobPath)
 }
 
+func (o *OCI) manifestBlobReaderAt(manifest ocispec.Manifest) (*os.File, error) {
+	blobPath, err := o.ensureBlob(string(manifest.Config.Digest.Algorithm().String()), manifest.Config.Digest.Hex())
+	if err != nil {
+		return nil, err
+	}
+	return os.Open(blobPath)
+}
+
 func (o *OCI) blobWriterAt(desc ocispec.Descriptor) (*os.File, error) {
 	blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex())
 	if err != nil {

pkg/cosign/cosign.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"encoding/json"
 	"time"
+	"bufio"
 
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"oras.land/oras-go/pkg/content"
@@ -69,53 +70,81 @@ func SaveImage(ctx context.Context, s *store.Layout, ref string) error {
 }
 
 // LoadImage loads store to a remote registry.
-func LoadImage(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error {
-	operation := func() error {
-		cosignBinaryPath, err := ensureCosignBinary(ctx, s)
-		if err != nil {
-			return err
-		}
+func LoadImages(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error {
+	l := log.FromContext(ctx)
 
-		cmd := exec.Command(cosignBinaryPath, "load", "--registry", registry, "--dir", s.Root)
+	cosignBinaryPath, err := ensureCosignBinary(ctx, s)
+	if err != nil {
+		return err
+	}
 
-		// Conditionally add extra registry flags.
-		if ropts.Insecure {
-			cmd.Args = append(cmd.Args, "--allow-insecure-registry=true")
-		}
-		if ropts.PlainHTTP {
-			cmd.Args = append(cmd.Args, "--allow-http-registry=true")
-		}
+	cmd := exec.Command(cosignBinaryPath, "load", "--registry", registry, "--dir", s.Root)
 
-		output, err := cmd.CombinedOutput()
-		if err != nil {
-			return fmt.Errorf("error copying store: %v, output: %s", err, output)
-		}
+	// Conditionally add extra registry flags.
+	if ropts.Insecure {
+		cmd.Args = append(cmd.Args, "--allow-insecure-registry=true")
+	}
+	if ropts.PlainHTTP {
+		cmd.Args = append(cmd.Args, "--allow-http-registry=true")
+	}
 
-		return nil
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+	// start the command after having set up the pipe
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	// read command's stdout line by line
+	output := bufio.NewScanner(stdout)
+	for output.Scan() {
+		l.Infof(output.Text()) // write each line to your log, or anything you need
+	}
+	if err := output.Err(); err != nil {
+		cmd.Wait()
+		return err
 	}
 
-	return RetryOperation(ctx, operation)
+	// read command's stderr line by line
+	errors := bufio.NewScanner(stderr)
+	for errors.Scan() {
+		l.Errorf(errors.Text()) // write each line to your log, or anything you need
+	}
+	if err := errors.Err(); err != nil {
+		cmd.Wait()
+		return err
+	}
+
+	// Wait for the command to finish
+	err = cmd.Wait()
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // RegistryLogin - performs cosign login
 func RegistryLogin(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error {
-	operation := func() error {
-		cosignBinaryPath, err := ensureCosignBinary(ctx, s)
-		if err != nil {
-			return err
-		}
+	cosignBinaryPath, err := ensureCosignBinary(ctx, s)
+	if err != nil {
+		return err
+	}
 
-		cmd := exec.Command(cosignBinaryPath, "login", registry, "-u", ropts.Username, "-p", ropts.Password)
+	cmd := exec.Command(cosignBinaryPath, "login", registry, "-u", ropts.Username, "-p", ropts.Password)
 
-		output, err := cmd.CombinedOutput()
-		if err != nil {
-			return fmt.Errorf("error logging into registry: %v, output: %s", err, output)
-		}
-
-		return nil
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error logging into registry: %v, output: %s", err, output)
 	}
 
-	return RetryOperation(ctx, operation)
+	return nil
 }
 
 func RetryOperation(ctx context.Context, operation func() error) error {