Merge branch 'multi' of https://github.com/umohnani8/common into mult…

…i-poc

Signed-off-by: Miloslav Trmač <[email protected]>

go.mod

@@ -52,18 +52,20 @@ require (
 	github.com/Microsoft/hcsshim v0.9.4 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/containerd/cgroups v1.0.4 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
 	github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/go-containerregistry v0.11.0 // indirect
 	github.com/google/go-intervals v0.0.2 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
@@ -73,25 +75,27 @@ require (
 	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/kr/fs v0.1.0 // indirect
-	github.com/letsencrypt/boulder v0.0.0-20220723181115-27de4befb95e // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/mistifyio/go-zfs/v3 v3.0.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/proglottis/gpgme v0.1.3 // indirect
+	github.com/prometheus/client_golang v1.12.2 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/sigstore/sigstore v1.4.2 // indirect
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
 	github.com/sylabs/sif/v2 v2.8.0 // indirect
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
-	github.com/theupdateframework/go-tuf v0.5.1 // indirect
-	github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/vbauerster/mpb/v7 v7.5.3 // indirect
@@ -103,6 +107,7 @@ require (
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 	google.golang.org/genproto v0.0.0-20220720214146-176da50484ac // indirect
 	google.golang.org/grpc v1.48.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
@@ -117,3 +122,5 @@ retract (
 )
 
 replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.1-0.20220617142545-8b9452f75cbc
+
+replace github.com/containers/image/v5 => github.com/umohnani8/image/v5 v5.0.0-20220309153911-e6ece2b1724f

libimage/load.go

@@ -53,7 +53,7 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) (
 			if err != nil {
 				return nil, ociArchiveTransport.Transport.Name(), err
 			}
-			images, err := r.copyFromDefault(ctx, ref, &options.CopyOptions)
+			images, err := r.loadMultiImageOCIArchive(ctx, ref, &options.CopyOptions)
 			return images, ociArchiveTransport.Transport.Name(), err
 		},
 
@@ -138,3 +138,22 @@ func (r *Runtime) loadMultiImageDockerArchive(ctx context.Context, ref types.Ima
 
 	return copiedImages, nil
 }
+
+func (r *Runtime) loadMultiImageOCIArchive(ctx context.Context, ref types.ImageReference, options *CopyOptions) ([]string, error) {
+	reader, err := ociArchiveTransport.NewReader(ctx, r.systemContextCopy(), ref)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := reader.Close(); err != nil {
+			logrus.Errorf(err.Error())
+		}
+	}()
+
+	copiedImages, err := r.copyFromOCIArchiveReader(ctx, reader, options)
+	if err != nil {
+		return nil, err
+	}
+
+	return copiedImages, nil
+}

libimage/load_test.go

@@ -45,6 +45,7 @@ func TestLoad(t *testing.T) {
 		{"testdata/oci-name-only.tar.gz", false, 1, []string{"localhost/pretty-empty:latest"}},
 		{"testdata/oci-non-docker-name.tar.gz", true, 0, nil},
 		{"testdata/oci-registry-name.tar.gz", false, 1, []string{"example.com/empty:latest"}},
+		{"testdata/oci-two-images.tar.xz", false, 2, []string{"example.com/empty:latest", "example.com/empty/but:different"}},
 		{"testdata/oci-unnamed.tar.gz", false, 1, []string{"sha256:5c8aca8137ac47e84c69ae93ce650ce967917cc001ba7aad5494073fac75b8b6"}},
 		{"testdata/buildkit-oci.tar", false, 1, []string{"github.com/buildkit/archive:oci"}},
 	} {

libimage/pull.go

@@ -150,6 +150,10 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
 	case dockerArchiveTransport.Transport.Name():
 		pulledImages, pullError = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)
 
+	// OCI ARCHIVE
+	case ociArchiveTransport.Transport.Name():
+		pulledImages, pullError = r.copyFromOCIArchive(ctx, ref, &options.CopyOptions)
+
 	// ALL OTHER TRANSPORTS
 	default:
 		pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
@@ -231,29 +235,6 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference,
 		storageName = toLocalImageName(split[0])
 		imageName = storageName
 
-	case ociArchiveTransport.Transport.Name():
-		manifestDescriptor, err := ociArchiveTransport.LoadManifestDescriptor(ref)
-		if err != nil {
-			return nil, err
-		}
-		storageName = nameFromAnnotations(manifestDescriptor.Annotations)
-		switch len(storageName) {
-		case 0:
-			// If there's no reference name in the annotations, compute an ID.
-			storageName, err = getImageID(ctx, ref, nil)
-			if err != nil {
-				return nil, err
-			}
-			imageName = "sha256:" + storageName[1:]
-		default:
-			named, err := NormalizeName(storageName)
-			if err != nil {
-				return nil, err
-			}
-			imageName = named.String()
-			storageName = imageName
-		}
-
 	case storageTransport.Transport.Name():
 		storageName = ref.StringWithinTransport()
 		named := ref.DockerReference()
@@ -367,6 +348,65 @@ func (r *Runtime) copyFromDockerArchiveReaderReference(ctx context.Context, read
 	return destNames, nil
 }
 
+func (r *Runtime) copyFromOCIArchive(ctx context.Context, readerRef types.ImageReference, options *CopyOptions) ([]string, error) {
+	reader, err := ociArchiveTransport.NewReader(ctx, &r.systemContext, readerRef)
+	if err != nil {
+		return nil, err
+	}
+	return r.copyFromOCIArchiveReader(ctx, reader, options)
+}
+
+func (r *Runtime) copyFromOCIArchiveReader(ctx context.Context, reader *ociArchiveTransport.Reader, options *CopyOptions) ([]string, error) {
+	c, err := r.newCopier(options)
+	if err != nil {
+		return nil, err
+	}
+	defer c.close()
+
+	list, err := reader.List()
+	if err != nil {
+		return nil, err
+	}
+
+	var names []string
+
+	for _, l := range list {
+		var storageName, imageName string
+
+		storageName = nameFromAnnotations(l.ManifestDescriptor.Annotations)
+		switch len(storageName) {
+		case 0:
+			// If there's no reference name in the annotations, compute an ID.
+			storageName, err = getImageID(ctx, l.ImageRef, nil)
+			if err != nil {
+				return nil, err
+			}
+			imageName = "sha256:" + storageName[1:]
+		default:
+			named, err := NormalizeName(storageName)
+			if err != nil {
+				return nil, err
+			}
+			imageName = named.String()
+			storageName = imageName
+		}
+
+		// Create a storage reference.
+		destRef, err := storageTransport.Transport.ParseStoreReference(r.store, storageName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "parsing %q", storageName)
+		}
+
+		if _, err := c.copy(ctx, l.ImageRef, destRef); err != nil {
+			return nil, err
+		}
+
+		names = append(names, imageName)
+	}
+
+	return names, nil
+}
+
 // copyFromRegistry pulls the specified, possibly unqualified, name from a
 // registry.  On successful pull it returns the ID of the image in local
 // storage.

libimage/save.go

@@ -11,7 +11,7 @@ import (
 	dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/manifest"
-	ociArchiveTransport "github.com/containers/image/v5/oci/archive"
+	"github.com/containers/image/v5/oci/archive"
 	ociTransport "github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
 	ociv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -47,8 +47,8 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
 	case 1:
 		// All formats support saving 1.
 	default:
-		if format != "docker-archive" {
-			return fmt.Errorf("unsupported format %q for saving multiple images (only docker-archive)", format)
+		if format != "docker-archive" && format != "oci-archive" {
+			return fmt.Errorf("unsupported format %q for saving multiple images (only docker-archive and oci-archive)", format)
 		}
 		if len(options.AdditionalTags) > 0 {
 			return fmt.Errorf("cannot save multiple images with multiple tags")
@@ -57,15 +57,17 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
 
 	// Dispatch the save operations.
 	switch format {
-	case "oci-archive", "oci-dir", "docker-dir":
+	case "oci-dir", "docker-dir":
 		if len(names) > 1 {
 			return fmt.Errorf("%q does not support saving multiple images (%v)", format, names)
 		}
 		return r.saveSingleImage(ctx, names[0], format, path, options)
-
 	case "docker-archive":
 		options.ManifestMIMEType = manifest.DockerV2Schema2MediaType
-		return r.saveDockerArchive(ctx, names, path, options)
+		return r.saveArchive(ctx, names, format, path, options)
+	case "oci-archive":
+		options.ManifestMIMEType = ociv1.MediaTypeImageManifest
+		return r.saveArchive(ctx, names, format, path, options)
 	}
 
 	return fmt.Errorf("unsupported format %q for saving images", format)
@@ -98,9 +100,6 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
 	// Prepare the destination reference.
 	var destRef types.ImageReference
 	switch format {
-	case "oci-archive":
-		destRef, err = ociArchiveTransport.NewReference(path, tag)
-
 	case "oci-dir":
 		destRef, err = ociTransport.NewReference(path, tag)
 		options.ManifestMIMEType = ociv1.MediaTypeImageManifest
@@ -127,17 +126,18 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
 	return err
 }
 
+type localImage struct {
+	image     *Image
+	tags      []reference.NamedTagged
+	destNames []string
+}
+
 // saveDockerArchive saves the specified images indicated by names to the path.
 // It loads all images from the local containers storage and assembles the meta
 // data needed to properly save images.  Since multiple names could refer to
 // the *same* image, we need to dance a bit and store additional "names".
 // Those can then be used as additional tags when copying.
-func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path string, options *SaveOptions) error {
-	type localImage struct {
-		image *Image
-		tags  []reference.NamedTagged
-	}
-
+func (r *Runtime) saveArchive(ctx context.Context, names []string, format, path string, options *SaveOptions) (finalErr error) {
 	additionalTags := []reference.NamedTagged{}
 	for _, tag := range options.AdditionalTags {
 		named, err := NormalizeName(tag)
@@ -180,18 +180,48 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
 			if withTag {
 				local.tags = append(local.tags, tagged)
 			}
+			local.destNames = append(local.destNames, tagged.String())
 		}
 		localImages[image.ID()] = local
 		if r.eventChannel != nil {
 			defer r.writeEvent(&Event{ID: image.ID(), Name: path, Time: time.Now(), Type: EventTypeImageSave})
 		}
 	}
 
+	switch format {
+	case "docker-archive":
+		if err := r.saveDockerArchive(ctx, path, orderedIDs, localImages, options); err != nil {
+			return err
+		}
+
+	case "oci-archive":
+		if err := r.saveOCIArchive(ctx, path, orderedIDs, localImages, options); err != nil {
+			return err
+		}
+
+	default:
+		return errors.Errorf("internal error: cannot save multiple images to format %q", format)
+	}
+
+	return nil
+}
+
+func (r *Runtime) saveDockerArchive(ctx context.Context, path string, orderedIDs []string, localImages map[string]*localImage, options *SaveOptions) (finalErr error) {
 	writer, err := dockerArchiveTransport.NewWriter(r.systemContextCopy(), path)
 	if err != nil {
 		return err
 	}
-	defer writer.Close()
+	defer func() {
+		err := writer.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+			return
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
 
 	for _, id := range orderedIDs {
 		local, exists := localImages[id]
@@ -222,6 +252,54 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
 			return err
 		}
 	}
+	return finalErr
+}
+
+func (r *Runtime) saveOCIArchive(ctx context.Context, path string, orderedIDs []string, localImages map[string]*localImage, options *SaveOptions) (finalErr error) {
+	writer, err := archive.NewWriter(ctx, r.systemContextCopy(), path)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := writer.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
 
-	return nil
+	for _, id := range orderedIDs {
+		local, exists := localImages[id]
+		if !exists {
+			return errors.Errorf("internal error: saveOCIArchive: ID %s not found in local map", id)
+		}
+
+		copyOpts := options.CopyOptions
+
+		c, err := r.newCopier(&copyOpts)
+		if err != nil {
+			return err
+		}
+		defer c.close()
+
+		for _, destName := range local.destNames {
+			destRef, err := writer.NewReference(destName)
+			if err != nil {
+				return err
+			}
+
+			srcRef, err := local.image.StorageReference()
+			if err != nil {
+				return err
+			}
+
+			if _, err := c.copy(ctx, srcRef, destRef); err != nil {
+				return err
+			}
+		}
+	}
+	return finalErr
 }