Skip to content

Commit

Permalink
Add multi oci manifest support to load and save
Browse files Browse the repository at this point in the history
Signed-off-by: Urvashi Mohnani <[email protected]>
  • Loading branch information
umohnani8 committed Mar 3, 2022
1 parent 9880eb4 commit c3d50f6
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 49 deletions.
21 changes: 20 additions & 1 deletion libimage/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) (
if err != nil {
return nil, ociArchiveTransport.Transport.Name(), err
}
images, err := r.copyFromDefault(ctx, ref, &options.CopyOptions)
images, err := r.loadMultiImageOCIArchive(ctx, ref, &options.CopyOptions)
return images, ociArchiveTransport.Transport.Name(), err
},

Expand Down Expand Up @@ -131,3 +131,22 @@ func (r *Runtime) loadMultiImageDockerArchive(ctx context.Context, ref types.Ima

return copiedImages, nil
}

func (r *Runtime) loadMultiImageOCIArchive(ctx context.Context, ref types.ImageReference, options *CopyOptions) ([]string, error) {
reader, err := ociArchiveTransport.NewReader(ctx, r.systemContextCopy(), ref)
if err != nil {
return nil, err
}
defer func() {
if err := reader.Close(); err != nil {
logrus.Errorf(err.Error())
}
}()

copiedImages, err := r.copyFromOCIArchiveReader(ctx, reader, options)
if err != nil {
return nil, err
}

return copiedImages, nil
}
1 change: 1 addition & 0 deletions libimage/load_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func TestLoad(t *testing.T) {
{"testdata/oci-name-only.tar.gz", false, 1, []string{"localhost/pretty-empty:latest"}},
{"testdata/oci-non-docker-name.tar.gz", true, 0, nil},
{"testdata/oci-registry-name.tar.gz", false, 1, []string{"example.com/empty:latest"}},
{"testdata/oci-two-images.tar.xz", false, 2, []string{"example.com/empty:latest", "example.com/empty/but:different"}},
{"testdata/oci-unnamed.tar.gz", false, 1, []string{"sha256:5c8aca8137ac47e84c69ae93ce650ce967917cc001ba7aad5494073fac75b8b6"}},
{"testdata/buildkit-oci.tar", false, 1, []string{"github.com/buildkit/archive:oci"}},
} {
Expand Down
86 changes: 63 additions & 23 deletions libimage/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
case dockerArchiveTransport.Transport.Name():
pulledImages, pullError = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)

// OCI ARCHIVE
case ociArchiveTransport.Transport.Name():
pulledImages, pullError = r.copyFromOCIArchive(ctx, ref, &options.CopyOptions)

// ALL OTHER TRANSPORTS
default:
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
Expand Down Expand Up @@ -212,29 +216,6 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference,
storageName = toLocalImageName(split[0])
imageName = storageName

case ociArchiveTransport.Transport.Name():
manifestDescriptor, err := ociArchiveTransport.LoadManifestDescriptor(ref)
if err != nil {
return nil, err
}
storageName = nameFromAnnotations(manifestDescriptor.Annotations)
switch len(storageName) {
case 0:
// If there's no reference name in the annotations, compute an ID.
storageName, err = getImageID(ctx, ref, nil)
if err != nil {
return nil, err
}
imageName = "sha256:" + storageName[1:]
default:
named, err := NormalizeName(storageName)
if err != nil {
return nil, err
}
imageName = named.String()
storageName = imageName
}

case storageTransport.Transport.Name():
storageName = ref.StringWithinTransport()
named := ref.DockerReference()
Expand Down Expand Up @@ -343,6 +324,65 @@ func (r *Runtime) copyFromDockerArchiveReaderReference(ctx context.Context, read
return destNames, nil
}

func (r *Runtime) copyFromOCIArchive(ctx context.Context, readerRef types.ImageReference, options *CopyOptions) ([]string, error) {
reader, err := ociArchiveTransport.NewReader(ctx, &r.systemContext, readerRef)
if err != nil {
return nil, err
}
return r.copyFromOCIArchiveReader(ctx, reader, options)
}

func (r *Runtime) copyFromOCIArchiveReader(ctx context.Context, reader *ociArchiveTransport.Reader, options *CopyOptions) ([]string, error) {
c, err := r.newCopier(options)
if err != nil {
return nil, err
}
defer c.close()

list, err := reader.List()
if err != nil {
return nil, err
}

var names []string

for _, l := range list {
var storageName, imageName string

storageName = nameFromAnnotations(l.ManifestDescriptor.Annotations)
switch len(storageName) {
case 0:
// If there's no reference name in the annotations, compute an ID.
storageName, err = getImageID(ctx, l.ImageRef, nil)
if err != nil {
return nil, err
}
imageName = "sha256:" + storageName[1:]
default:
named, err := NormalizeName(storageName)
if err != nil {
return nil, err
}
imageName = named.String()
storageName = imageName
}

// Create a storage reference.
destRef, err := storageTransport.Transport.ParseStoreReference(r.store, storageName)
if err != nil {
return nil, errors.Wrapf(err, "parsing %q", storageName)
}

if _, err := c.copy(ctx, l.ImageRef, destRef); err != nil {
return nil, err
}

names = append(names, imageName)
}

return names, nil
}

// copyFromRegistry pulls the specified, possibly unqualified, name from a
// registry. On successful pull it returns the ID of the image in local
// storage.
Expand Down
112 changes: 95 additions & 17 deletions libimage/save.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/manifest"
ociArchiveTransport "github.com/containers/image/v5/oci/archive"
"github.com/containers/image/v5/oci/archive"
ociTransport "github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/types"
ociv1 "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -46,8 +46,8 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
case 1:
// All formats support saving 1.
default:
if format != "docker-archive" {
return errors.Errorf("unsupported format %q for saving multiple images (only docker-archive)", format)
if format != "docker-archive" && format != "oci-archive" {
return errors.Errorf("unsupported format %q for saving multiple images (only docker-archive and oci-archive)", format)
}
if len(options.AdditionalTags) > 0 {
return errors.Errorf("cannot save multiple images with multiple tags")
Expand All @@ -56,15 +56,17 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,

// Dispatch the save operations.
switch format {
case "oci-archive", "oci-dir", "docker-dir":
case "oci-dir", "docker-dir":
if len(names) > 1 {
return errors.Errorf("%q does not support saving multiple images (%v)", format, names)
}
return r.saveSingleImage(ctx, names[0], format, path, options)

case "docker-archive":
options.ManifestMIMEType = manifest.DockerV2Schema2MediaType
return r.saveDockerArchive(ctx, names, path, options)
return r.saveArchive(ctx, names, format, path, options)
case "oci-archive":
options.ManifestMIMEType = ociv1.MediaTypeImageManifest
return r.saveArchive(ctx, names, format, path, options)
}

return errors.Errorf("unsupported format %q for saving images", format)
Expand Down Expand Up @@ -98,9 +100,6 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
// Prepare the destination reference.
var destRef types.ImageReference
switch format {
case "oci-archive":
destRef, err = ociArchiveTransport.NewReference(path, tag)

case "oci-dir":
destRef, err = ociTransport.NewReference(path, tag)
options.ManifestMIMEType = ociv1.MediaTypeImageManifest
Expand All @@ -127,17 +126,18 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
return err
}

type localImage struct {
image *Image
tags []reference.NamedTagged
destNames []string
}

// saveDockerArchive saves the specified images indicated by names to the path.
// It loads all images from the local containers storage and assembles the meta
// data needed to properly save images. Since multiple names could refer to
// the *same* image, we need to dance a bit and store additional "names".
// Those can then be used as additional tags when copying.
func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path string, options *SaveOptions) error {
type localImage struct {
image *Image
tags []reference.NamedTagged
}

func (r *Runtime) saveArchive(ctx context.Context, names []string, format, path string, options *SaveOptions) (finalErr error) {
additionalTags := []reference.NamedTagged{}
for _, tag := range options.AdditionalTags {
named, err := NormalizeName(tag)
Expand Down Expand Up @@ -180,18 +180,48 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
if withTag {
local.tags = append(local.tags, tagged)
}
local.destNames = append(local.destNames, tagged.String())
}
localImages[image.ID()] = local
if r.eventChannel != nil {
defer r.writeEvent(&Event{ID: image.ID(), Name: path, Time: time.Now(), Type: EventTypeImageSave})
}
}

switch format {
case "docker-archive":
if err := r.saveDockerArchive(ctx, path, orderedIDs, localImages, options); err != nil {
return err
}

case "oci-archive":
if err := r.saveOCIArchive(ctx, path, orderedIDs, localImages, options); err != nil {
return err
}

default:
return errors.Errorf("internal error: cannot save multiple images to format %q", format)
}

return nil
}

func (r *Runtime) saveDockerArchive(ctx context.Context, path string, orderedIDs []string, localImages map[string]*localImage, options *SaveOptions) (finalErr error) {
writer, err := dockerArchiveTransport.NewWriter(r.systemContextCopy(), path)
if err != nil {
return err
}
defer writer.Close()
defer func() {
err := writer.Close()
if err == nil {
return
}
if finalErr == nil {
finalErr = err
return
}
finalErr = errors.Wrap(finalErr, err.Error())
}()

for _, id := range orderedIDs {
local, exists := localImages[id]
Expand Down Expand Up @@ -222,6 +252,54 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
return err
}
}
return finalErr
}

func (r *Runtime) saveOCIArchive(ctx context.Context, path string, orderedIDs []string, localImages map[string]*localImage, options *SaveOptions) (finalErr error) {
writer, err := archive.NewWriter(ctx, r.systemContextCopy(), path)
if err != nil {
return err
}
defer func() {
err := writer.Close()
if err == nil {
return
}
if finalErr == nil {
finalErr = err
}
finalErr = errors.Wrap(finalErr, err.Error())
}()

return nil
for _, id := range orderedIDs {
local, exists := localImages[id]
if !exists {
return errors.Errorf("internal error: saveOCIArchive: ID %s not found in local map", id)
}

copyOpts := options.CopyOptions

c, err := r.newCopier(&copyOpts)
if err != nil {
return err
}
defer c.close()

for _, destName := range local.destNames {
destRef, err := writer.NewReference(destName)
if err != nil {
return err
}

srcRef, err := local.image.StorageReference()
if err != nil {
return err
}

if _, err := c.copy(ctx, srcRef, destRef); err != nil {
return err
}
}
}
return finalErr
}
31 changes: 23 additions & 8 deletions libimage/save_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,20 @@ func TestSave(t *testing.T) {
// reload the images for each test.
saveOptions := &SaveOptions{}
saveOptions.Writer = os.Stdout
imageCache, err := ioutil.TempFile("", "saveimagecache")
dockerImageCache, err := ioutil.TempFile("", "savedockerimagecache")
require.NoError(t, err)
imageCache.Close()
defer os.Remove(imageCache.Name())
err = runtime.Save(ctx, []string{"alpine", "busybox"}, "docker-archive", imageCache.Name(), saveOptions)
dockerImageCache.Close()
defer os.Remove(dockerImageCache.Name())
err = runtime.Save(ctx, []string{"alpine", "busybox"}, "docker-archive", dockerImageCache.Name(), saveOptions)
require.NoError(t, err)

saveOptions = &SaveOptions{}
saveOptions.Writer = os.Stdout
ociImageCache, err := ioutil.TempFile("", "saveociimagecache")
require.NoError(t, err)
ociImageCache.Close()
defer os.Remove(ociImageCache.Name())
err = runtime.Save(ctx, []string{"alpine", "busybox"}, "oci-archive", ociImageCache.Name(), saveOptions)
require.NoError(t, err)

loadOptions := &LoadOptions{}
Expand All @@ -56,8 +65,9 @@ func TestSave(t *testing.T) {
// oci
{[]string{"busybox"}, nil, "oci-dir", true, false},
{[]string{"busybox"}, nil, "oci-archive", false, false},
// oci-archive doesn't support multi-image archives
{[]string{"busybox", "alpine"}, nil, "oci-archive", false, true},
{[]string{"busybox", "alpine"}, nil, "oci-archive", false, false},
// additional tags and multi-images conflict
{[]string{"busybox", "alpine"}, []string{"tag"}, "oci-archive", false, true},
// docker
{[]string{"busybox"}, nil, "docker-archive", false, false},
{[]string{"busybox"}, []string{"localhost/tag:1", "quay.io/repo/image:tag"}, "docker-archive", false, false},
Expand All @@ -69,8 +79,13 @@ func TestSave(t *testing.T) {
// First clean up all images and load the cache.
_, rmErrors := runtime.RemoveImages(ctx, nil, nil)
require.Nil(t, rmErrors)
_, err = runtime.Load(ctx, imageCache.Name(), loadOptions)
require.NoError(t, err)
if test.format == "oci-archive" {
_, err = runtime.Load(ctx, ociImageCache.Name(), loadOptions)
require.NoError(t, err)
} else {
_, err = runtime.Load(ctx, dockerImageCache.Name(), loadOptions)
require.NoError(t, err)
}

tmp, err := ioutil.TempDir("", "libimagesavetest")
require.NoError(t, err)
Expand Down
Binary file added libimage/testdata/oci-two-images.tar.xz
Binary file not shown.

0 comments on commit c3d50f6

Please sign in to comment.