Add multi oci manifest support to load and save

Signed-off-by: Urvashi Mohnani <[email protected]>

libimage/load.go

@@ -53,7 +53,7 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) (
 			if err != nil {
 				return nil, ociArchiveTransport.Transport.Name(), err
 			}
-			images, err := r.copyFromDefault(ctx, ref, &options.CopyOptions)
+			images, err := r.loadMultiImageOCIArchive(ctx, ref, &options.CopyOptions)
 			return images, ociArchiveTransport.Transport.Name(), err
 		},
 
@@ -131,3 +131,22 @@ func (r *Runtime) loadMultiImageDockerArchive(ctx context.Context, ref types.Ima
 
 	return copiedImages, nil
 }
+
+func (r *Runtime) loadMultiImageOCIArchive(ctx context.Context, ref types.ImageReference, options *CopyOptions) ([]string, error) {
+	reader, err := ociArchiveTransport.NewReader(ctx, r.systemContextCopy(), ref)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := reader.Close(); err != nil {
+			logrus.Errorf(err.Error())
+		}
+	}()
+
+	copiedImages, err := r.copyFromOCIArchiveReader(ctx, reader, options)
+	if err != nil {
+		return nil, err
+	}
+
+	return copiedImages, nil
+}

libimage/load_test.go

@@ -33,6 +33,8 @@ func TestLoad(t *testing.T) {
 		{"testdata/oci-name-only.tar.gz", false, 1, []string{"localhost/pretty-empty:latest"}},
 		{"testdata/oci-non-docker-name.tar.gz", true, 0, nil},
 		{"testdata/oci-registry-name.tar.gz", false, 1, []string{"example.com/empty:latest"}},
+		// {"testdata/oci-two-names.tar.xz", false, 1, []string{"localhost/pretty-empty:latest", "example.com/empty:latest"}},
+		{"testdata/oci-two-images.tar.xz", false, 2, []string{"example.com/empty:latest", "example.com/empty/but:different"}},
 		{"testdata/oci-unnamed.tar.gz", false, 1, []string{"sha256:5c8aca8137ac47e84c69ae93ce650ce967917cc001ba7aad5494073fac75b8b6"}},
 		{"testdata/buildkit-oci.tar", false, 1, []string{"github.com/buildkit/archive:oci"}},
 	} {

libimage/pull.go

@@ -150,6 +150,10 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
 	case dockerArchiveTransport.Transport.Name():
 		pulledImages, pullError = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)
 
+	// OCI ARCHIVE
+	case ociArchiveTransport.Transport.Name():
+		pulledImages, pullError = r.copyFromOCIArchive(ctx, ref, &options.CopyOptions)
+
 	// ALL OTHER TRANSPORTS
 	default:
 		pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
@@ -212,29 +216,6 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference,
 		storageName = toLocalImageName(split[0])
 		imageName = storageName
 
-	case ociArchiveTransport.Transport.Name():
-		manifestDescriptor, err := ociArchiveTransport.LoadManifestDescriptor(ref)
-		if err != nil {
-			return nil, err
-		}
-		storageName = nameFromAnnotations(manifestDescriptor.Annotations)
-		switch len(storageName) {
-		case 0:
-			// If there's no reference name in the annotations, compute an ID.
-			storageName, err = getImageID(ctx, ref, nil)
-			if err != nil {
-				return nil, err
-			}
-			imageName = "sha256:" + storageName[1:]
-		default:
-			named, err := NormalizeName(storageName)
-			if err != nil {
-				return nil, err
-			}
-			imageName = named.String()
-			storageName = imageName
-		}
-
 	case storageTransport.Transport.Name():
 		storageName = ref.StringWithinTransport()
 		named := ref.DockerReference()
@@ -343,6 +324,65 @@ func (r *Runtime) copyFromDockerArchiveReaderReference(ctx context.Context, read
 	return destNames, nil
 }
 
+func (r *Runtime) copyFromOCIArchive(ctx context.Context, readerRef types.ImageReference, options *CopyOptions) ([]string, error) {
+	reader, err := ociArchiveTransport.NewReader(ctx, &r.systemContext, readerRef)
+	if err != nil {
+		return nil, err
+	}
+	return r.copyFromOCIArchiveReader(ctx, reader, options)
+}
+
+func (r *Runtime) copyFromOCIArchiveReader(ctx context.Context, reader *ociArchiveTransport.Reader, options *CopyOptions) ([]string, error) {
+	c, err := r.newCopier(options)
+	if err != nil {
+		return nil, err
+	}
+	defer c.close()
+
+	list, err := reader.List()
+	if err != nil {
+		return nil, err
+	}
+
+	var names []string
+
+	for _, l := range list {
+		var storageName, imageName string
+
+		storageName = nameFromAnnotations(l.ManifestDescriptor.Annotations)
+		switch len(storageName) {
+		case 0:
+			// If there's no reference name in the annotations, compute an ID.
+			storageName, err = getImageID(ctx, l.ImageRef, nil)
+			if err != nil {
+				return nil, err
+			}
+			imageName = "sha256:" + storageName[1:]
+		default:
+			named, err := NormalizeName(storageName)
+			if err != nil {
+				return nil, err
+			}
+			imageName = named.String()
+			storageName = imageName
+		}
+
+		// Create a storage reference.
+		destRef, err := storageTransport.Transport.ParseStoreReference(r.store, storageName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "parsing %q", storageName)
+		}
+
+		if _, err := c.copy(ctx, l.ImageRef, destRef); err != nil {
+			return nil, err
+		}
+
+		names = append(names, imageName)
+	}
+
+	return names, nil
+}
+
 // copyFromRegistry pulls the specified, possibly unqualified, name from a
 // registry.  On successful pull it returns the ID of the image in local
 // storage.

libimage/save.go

@@ -9,7 +9,7 @@ import (
 	dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/manifest"
-	ociArchiveTransport "github.com/containers/image/v5/oci/archive"
+	"github.com/containers/image/v5/oci/archive"
 	ociTransport "github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
 	ociv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -46,8 +46,8 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
 	case 1:
 		// All formats support saving 1.
 	default:
-		if format != "docker-archive" {
-			return errors.Errorf("unsupported format %q for saving multiple images (only docker-archive)", format)
+		if format != "docker-archive" && format != "oci-archive" {
+			return errors.Errorf("unsupported format %q for saving multiple images (only docker-archive and oci-archive)", format)
 		}
 		if len(options.AdditionalTags) > 0 {
 			return errors.Errorf("cannot save multiple images with multiple tags")
@@ -56,15 +56,17 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
 
 	// Dispatch the save operations.
 	switch format {
-	case "oci-archive", "oci-dir", "docker-dir":
+	case "oci-dir", "docker-dir":
 		if len(names) > 1 {
 			return errors.Errorf("%q does not support saving multiple images (%v)", format, names)
 		}
 		return r.saveSingleImage(ctx, names[0], format, path, options)
-
 	case "docker-archive":
 		options.ManifestMIMEType = manifest.DockerV2Schema2MediaType
-		return r.saveDockerArchive(ctx, names, path, options)
+		return r.saveArchive(ctx, names, format, path, options)
+	case "oci-archive":
+		options.ManifestMIMEType = ociv1.MediaTypeImageManifest
+		return r.saveArchive(ctx, names, format, path, options)
 	}
 
 	return errors.Errorf("unsupported format %q for saving images", format)
@@ -98,9 +100,6 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
 	// Prepare the destination reference.
 	var destRef types.ImageReference
 	switch format {
-	case "oci-archive":
-		destRef, err = ociArchiveTransport.NewReference(path, tag)
-
 	case "oci-dir":
 		destRef, err = ociTransport.NewReference(path, tag)
 		options.ManifestMIMEType = ociv1.MediaTypeImageManifest
@@ -127,17 +126,18 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
 	return err
 }
 
+type localImage struct {
+	image     *Image
+	tags      []reference.NamedTagged
+	destNames []string
+}
+
 // saveDockerArchive saves the specified images indicated by names to the path.
 // It loads all images from the local containers storage and assembles the meta
 // data needed to properly save images.  Since multiple names could refer to
 // the *same* image, we need to dance a bit and store additional "names".
 // Those can then be used as additional tags when copying.
-func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path string, options *SaveOptions) error {
-	type localImage struct {
-		image *Image
-		tags  []reference.NamedTagged
-	}
-
+func (r *Runtime) saveArchive(ctx context.Context, names []string, format, path string, options *SaveOptions) (finalErr error) {
 	additionalTags := []reference.NamedTagged{}
 	for _, tag := range options.AdditionalTags {
 		named, err := NormalizeName(tag)
@@ -180,18 +180,47 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
 			if withTag {
 				local.tags = append(local.tags, tagged)
 			}
+			local.destNames = append(local.destNames, tagged.String())
 		}
 		localImages[image.ID()] = local
 		if r.eventChannel != nil {
 			defer r.writeEvent(&Event{ID: image.ID(), Name: path, Time: time.Now(), Type: EventTypeImageSave})
 		}
 	}
 
+	switch format {
+	case "docker-archive":
+		if err := r.copyDockerArchive(ctx, path, orderedIDs, localImages, options); err != nil {
+			return err
+		}
+
+	case "oci-archive":
+		if err := r.copyOCIArchive(ctx, path, orderedIDs, localImages, options); err != nil {
+			return err
+		}
+	default:
+		return errors.Errorf("internal error: cannot save multiple images to format %q", format)
+	}
+
+	return nil
+}
+
+func (r *Runtime) copyDockerArchive(ctx context.Context, path string, orderedIDs []string, localImages map[string]*localImage, options *SaveOptions) (finalErr error) {
 	writer, err := dockerArchiveTransport.NewWriter(r.systemContextCopy(), path)
 	if err != nil {
 		return err
 	}
-	defer writer.Close()
+	defer func() {
+		err := writer.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+			return
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
 
 	for _, id := range orderedIDs {
 		local, exists := localImages[id]
@@ -222,6 +251,54 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
 			return err
 		}
 	}
+	return finalErr
+}
 
-	return nil
+func (r *Runtime) copyOCIArchive(ctx context.Context, path string, orderedIDs []string, localImages map[string]*localImage, options *SaveOptions) (finalErr error) {
+	writer, err := archive.NewWriter(ctx, r.systemContextCopy(), path)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := writer.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
+
+	for _, id := range orderedIDs {
+		local, exists := localImages[id]
+		if !exists {
+			return errors.Errorf("internal error: saveOCIArchive: ID %s not found in local map", id)
+		}
+
+		copyOpts := options.CopyOptions
+
+		c, err := r.newCopier(&copyOpts)
+		if err != nil {
+			return err
+		}
+		defer c.close()
+
+		for _, destName := range local.destNames {
+			destRef, err := writer.NewReference(destName)
+			if err != nil {
+				return err
+			}
+
+			srcRef, err := local.image.StorageReference()
+			if err != nil {
+				return err
+			}
+
+			if _, err := c.copy(ctx, srcRef, destRef); err != nil {
+				return err
+			}
+		}
+	}
+	return finalErr
 }

libimage/save_test.go

@@ -28,11 +28,20 @@ func TestSave(t *testing.T) {
 	// reload the images for each test.
 	saveOptions := &SaveOptions{}
 	saveOptions.Writer = os.Stdout
-	imageCache, err := ioutil.TempFile("", "saveimagecache")
+	dockerImageCache, err := ioutil.TempFile("", "savedockerimagecache")
 	require.NoError(t, err)
-	imageCache.Close()
-	defer os.Remove(imageCache.Name())
-	err = runtime.Save(ctx, []string{"alpine", "busybox"}, "docker-archive", imageCache.Name(), saveOptions)
+	dockerImageCache.Close()
+	defer os.Remove(dockerImageCache.Name())
+	err = runtime.Save(ctx, []string{"alpine", "busybox"}, "docker-archive", dockerImageCache.Name(), saveOptions)
+	require.NoError(t, err)
+
+	saveOptions = &SaveOptions{}
+	saveOptions.Writer = os.Stdout
+	ociImageCache, err := ioutil.TempFile("", "saveociimagecache")
+	require.NoError(t, err)
+	ociImageCache.Close()
+	defer os.Remove(ociImageCache.Name())
+	err = runtime.Save(ctx, []string{"alpine", "busybox"}, "oci-archive", ociImageCache.Name(), saveOptions)
 	require.NoError(t, err)
 
 	loadOptions := &LoadOptions{}
@@ -56,8 +65,10 @@ func TestSave(t *testing.T) {
 		// oci
 		{[]string{"busybox"}, nil, "oci-dir", true, false},
 		{[]string{"busybox"}, nil, "oci-archive", false, false},
-		// oci-archive doesn't support multi-image archives
-		{[]string{"busybox", "alpine"}, nil, "oci-archive", false, true},
+		// {[]string{"busybox"}, []string{"localhost/tag:1", "quay.io/repo/image:tag"}, "oci-archive", false, false},
+		{[]string{"busybox", "alpine"}, nil, "oci-archive", false, false},
+		// additional tags and multi-images conflict
+		{[]string{"busybox", "alpine"}, []string{"tag"}, "oci-archive", false, true},
 		// docker
 		{[]string{"busybox"}, nil, "docker-archive", false, false},
 		{[]string{"busybox"}, []string{"localhost/tag:1", "quay.io/repo/image:tag"}, "docker-archive", false, false},
@@ -69,8 +80,13 @@ func TestSave(t *testing.T) {
 		// First clean up all images and load the cache.
 		_, rmErrors := runtime.RemoveImages(ctx, nil, nil)
 		require.Nil(t, rmErrors)
-		_, err = runtime.Load(ctx, imageCache.Name(), loadOptions)
-		require.NoError(t, err)
+		if test.format == "oci-archive" {
+			_, err = runtime.Load(ctx, ociImageCache.Name(), loadOptions)
+			require.NoError(t, err)
+		} else {
+			_, err = runtime.Load(ctx, dockerImageCache.Name(), loadOptions)
+			require.NoError(t, err)
+		}
 
 		tmp, err := ioutil.TempDir("", "libimagesavetest")
 		require.NoError(t, err)