Add reader/writer for oci-archive multi image support

Add reader/writer with helpers to allow podman save/load multi oci-archive images.
Allow read oci-archive using source_index to point to the index from oci-archive manifest.
Also reimplement ociArchiveImage{Source,Destination} to support this.

Signed-off-by: Qi Wang <[email protected]>
Signed-off-by: Urvashi Mohnani <[email protected]>

docs/containers-transports.5.md

@@ -61,10 +61,12 @@ The _algo:digest_ refers to the image ID reported by docker-inspect(1).
 
 An image compliant with the "Open Container Image Layout Specification" at _path_.
 Using a _reference_ is optional and allows for storing multiple images at the same _path_.
+For reading images, @_source-index_ is a zero-based index in manifest (to access untagged images).
+If neither tag nor @_source_index is specified when reading an image, the path must contain exactly one image.
 
 ### **oci-archive:**_path[:reference]_
 
-An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.
+An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.  For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images).  If neither tag nor @_source_index is specified when reading an archive, the archive must contain exactly one image.
 
 ### **ostree:**_docker-reference[@/absolute/repo/path]_
 

oci/archive/oci_dest.go

@@ -13,16 +13,30 @@ import (
 )
 
 type ociArchiveImageDestination struct {
-	ref          ociArchiveReference
-	unpackedDest types.ImageDestination
-	tempDirRef   tempDirOCIRef
+	ref              ociArchiveReference
+	unpackedDest     types.ImageDestination
+	tempDirRef       tempDirOCIRef
+	archiveWriterRef *tempDirOCIRef
 }
 
 // newImageDestination returns an ImageDestination for writing to an existing directory.
-func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
-	tempDirRef, err := createOCIRef(sys, ref.image)
-	if err != nil {
-		return nil, errors.Wrapf(err, "creating oci reference")
+func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference, archiveWriterRef *tempDirOCIRef) (types.ImageDestination, error) {
+	var (
+		tempDirRef tempDirOCIRef
+		err        error
+	)
+
+	if ref.sourceIndex != -1 {
+		return nil, errors.Errorf("destination reference must not contain a manifest index @%d", ref.sourceIndex)
+	}
+
+	if archiveWriterRef != nil {
+		tempDirRef = *archiveWriterRef
+	} else {
+		tempDirRef, err = createOCIRef(sys, ref.image, -1)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error creating oci reference")
+		}
 	}
 	unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
 	if err != nil {
@@ -32,8 +46,9 @@ func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociA
 		return nil, err
 	}
 	return &ociArchiveImageDestination{ref: ref,
-		unpackedDest: unpackedDest,
-		tempDirRef:   tempDirRef}, nil
+		unpackedDest:     unpackedDest,
+		archiveWriterRef: archiveWriterRef,
+		tempDirRef:       tempDirRef}, nil
 }
 
 // Reference returns the reference used to set up this destination.
@@ -44,6 +59,9 @@ func (d *ociArchiveImageDestination) Reference() types.ImageReference {
 // Close removes resources associated with an initialized ImageDestination, if any
 // Close deletes the temp directory of the oci-archive image
 func (d *ociArchiveImageDestination) Close() error {
+	if d.archiveWriterRef != nil {
+		return nil
+	}
 	defer func() {
 		err := d.tempDirRef.deleteTempDir()
 		logrus.Debugf("Error deleting temporary directory: %v", err)
@@ -138,6 +156,10 @@ func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedTopleve
 		return errors.Wrapf(err, "storing image %q", d.ref.image)
 	}
 
+	if d.archiveWriterRef != nil {
+		return nil
+	}
+
 	// path of directory to tar up
 	src := d.tempDirRef.tempDirectory
 	// path to save tarred up file

oci/archive/oci_src.go

@@ -13,19 +13,27 @@ import (
 )
 
 type ociArchiveImageSource struct {
-	ref         ociArchiveReference
-	unpackedSrc types.ImageSource
-	tempDirRef  tempDirOCIRef
+	ref              ociArchiveReference
+	unpackedSrc      types.ImageSource
+	tempDirRef       tempDirOCIRef
+	archiveReaderRef *tempDirOCIRef
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
 // newImageSource untars the file and saves it in a temp directory
-func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
-	tempDirRef, err := createUntarTempDir(sys, ref)
-	if err != nil {
-		return nil, errors.Wrap(err, "creating temp directory")
+func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference, archiveReaderRef *tempDirOCIRef) (types.ImageSource, error) {
+	var (
+		tempDirRef tempDirOCIRef
+		err        error
+	)
+	if archiveReaderRef != nil {
+		tempDirRef = *archiveReaderRef
+	} else {
+		tempDirRef, err = createUntarTempDir(sys, ref)
+		if err != nil {
+			return nil, errors.Wrap(err, "error creating temp directory")
+		}
 	}
-
 	unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
 	if err != nil {
 		if err := tempDirRef.deleteTempDir(); err != nil {
@@ -34,8 +42,9 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiv
 		return nil, err
 	}
 	return &ociArchiveImageSource{ref: ref,
-		unpackedSrc: unpackedSrc,
-		tempDirRef:  tempDirRef}, nil
+		unpackedSrc:      unpackedSrc,
+		archiveReaderRef: archiveReaderRef,
+		tempDirRef:       tempDirRef}, nil
 }
 
 // LoadManifestDescriptor loads the manifest
@@ -74,6 +83,9 @@ func (s *ociArchiveImageSource) Reference() types.ImageReference {
 // Close removes resources associated with an initialized ImageSource, if any.
 // Close deletes the temporary directory at dst
 func (s *ociArchiveImageSource) Close() error {
+	if s.archiveReaderRef != nil {
+		return nil
+	}
 	defer func() {
 		err := s.tempDirRef.deleteTempDir()
 		logrus.Debugf("error deleting tmp dir: %v", err)

oci/archive/oci_transport.go

@@ -32,9 +32,12 @@ type ociArchiveTransport struct{}
 
 // ociArchiveReference is an ImageReference for OCI Archive paths
 type ociArchiveReference struct {
-	file         string
-	resolvedFile string
-	image        string
+	file             string
+	resolvedFile     string
+	image            string
+	sourceIndex      int
+	archiveReaderRef *tempDirOCIRef
+	archiveWriterRef *tempDirOCIRef
 }
 
 func (t ociArchiveTransport) Name() string {
@@ -54,12 +57,24 @@ func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) erro
 
 // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference.
 func ParseReference(reference string) (types.ImageReference, error) {
-	file, image := internal.SplitPathAndImage(reference)
-	return NewReference(file, image)
+	file, image, index, err := internal.ParseReferenceIntoElements(reference)
+	if err != nil {
+		return nil, err
+	}
+	return newReference(file, image, index, nil, nil)
 }
 
-// NewReference returns an OCI reference for a file and a image.
+// NewReference returns an OCI reference for a file and an image.
 func NewReference(file, image string) (types.ImageReference, error) {
+	return newReference(file, image, -1, nil, nil)
+}
+
+// NewIndexReference returns an OCI reference for a file and sourecIndex points to the image.
+func NewIndexReference(file string, sourceIndex int) (types.ImageReference, error) {
+	return newReference(file, "", sourceIndex, nil, nil)
+}
+
+func newReference(file, image string, sourceIndex int, archiveReaderRef, archiveWriterRef *tempDirOCIRef) (types.ImageReference, error) {
 	resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file)
 	if err != nil {
 		return nil, err
@@ -73,7 +88,13 @@ func NewReference(file, image string) (types.ImageReference, error) {
 		return nil, err
 	}
 
-	return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil
+	if sourceIndex != -1 && sourceIndex < 0 {
+		return nil, errors.Errorf("Invalid oci archive: reference: index @%d must not be negative", sourceIndex)
+	}
+	if sourceIndex != -1 && image != "" {
+		return nil, errors.Errorf("Can not set image %s and index @%d at the same time", image, sourceIndex)
+	}
+	return ociArchiveReference{file: file, resolvedFile: resolved, image: image, sourceIndex: sourceIndex, archiveReaderRef: archiveReaderRef, archiveWriterRef: archiveWriterRef}, nil
 }
 
 func (ref ociArchiveReference) Transport() types.ImageTransport {
@@ -83,7 +104,10 @@ func (ref ociArchiveReference) Transport() types.ImageTransport {
 // StringWithinTransport returns a string representation of the reference, which MUST be such that
 // reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
 func (ref ociArchiveReference) StringWithinTransport() string {
-	return fmt.Sprintf("%s:%s", ref.file, ref.image)
+	if ref.sourceIndex == -1 {
+		return fmt.Sprintf("%s:%s", ref.file, ref.image)
+	}
+	return fmt.Sprintf("%s:@%d", ref.file, ref.sourceIndex)
 }
 
 // DockerReference returns a Docker reference associated with this reference
@@ -123,7 +147,7 @@ func (ref ociArchiveReference) PolicyConfigurationNamespaces() []string {
 // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
 // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details.
 func (ref ociArchiveReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
-	src, err := newImageSource(ctx, sys, ref)
+	src, err := newImageSource(ctx, sys, ref, ref.archiveReaderRef)
 	if err != nil {
 		return nil, err
 	}
@@ -133,13 +157,13 @@ func (ref ociArchiveReference) NewImage(ctx context.Context, sys *types.SystemCo
 // NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
 func (ref ociArchiveReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) {
-	return newImageSource(ctx, sys, ref)
+	return newImageSource(ctx, sys, ref, ref.archiveReaderRef)
 }
 
 // NewImageDestination returns a types.ImageDestination for this reference.
 // The caller must call .Close() on the returned ImageDestination.
 func (ref ociArchiveReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) {
-	return newImageDestination(ctx, sys, ref)
+	return newImageDestination(ctx, sys, ref, ref.archiveWriterRef)
 }
 
 // DeleteImage deletes the named image from the registry, if supported.
@@ -160,14 +184,20 @@ func (t *tempDirOCIRef) deleteTempDir() error {
 
 // createOCIRef creates the oci reference of the image
 // If SystemContext.BigFilesTemporaryDir not "", overrides the temporary directory to use for storing big files
-func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) {
+func createOCIRef(sys *types.SystemContext, image string, sourceIndex int) (tempDirOCIRef, error) {
 	dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci")
 	if err != nil {
 		return tempDirOCIRef{}, errors.Wrapf(err, "creating temp directory")
 	}
-	ociRef, err := ocilayout.NewReference(dir, image)
-	if err != nil {
-		return tempDirOCIRef{}, err
+	var ociRef types.ImageReference
+	if sourceIndex > -1 {
+		if ociRef, err = ocilayout.NewIndexReference(dir, sourceIndex); err != nil {
+			return tempDirOCIRef{}, err
+		}
+	} else {
+		if ociRef, err = ocilayout.NewReference(dir, image); err != nil {
+			return tempDirOCIRef{}, err
+		}
 	}
 
 	tempDirRef := tempDirOCIRef{tempDirectory: dir, ociRefExtracted: ociRef}
@@ -176,7 +206,7 @@ func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error)
 
 // creates the temporary directory and copies the tarred content to it
 func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (tempDirOCIRef, error) {
-	tempDirRef, err := createOCIRef(sys, ref.image)
+	tempDirRef, err := createOCIRef(sys, ref.image, ref.sourceIndex)
 	if err != nil {
 		return tempDirOCIRef{}, errors.Wrap(err, "creating oci reference")
 	}

oci/archive/oci_transport_test.go

@@ -60,11 +60,18 @@ func testParseReference(t *testing.T, fn func(string) (types.ImageReference, err
 		"relativepath",
 		tmpDir + "/thisdoesnotexist",
 	} {
-		for _, image := range []struct{ suffix, image string }{
-			{":notlatest:image", "notlatest:image"},
-			{":latestimage", "latestimage"},
-			{":", ""},
-			{"", ""},
+		for _, image := range []struct {
+			suffix, image       string
+			expectedSourceIndex int
+		}{
+			{":notlatest:image", "notlatest:image", -1},
+			{":latestimage", "latestimage", -1},
+			{":busybox@0", "busybox@0", -1},
+			{":", "", -1}, // No Image
+			{"", "", -1},
+			{":@0", "", 0}, // Explicit sourceIndex of image
+			{":@10", "", 10},
+			{":@999999", "", 999999},
 		} {
 			input := path + image.suffix
 			ref, err := fn(input)
@@ -73,11 +80,23 @@ func testParseReference(t *testing.T, fn func(string) (types.ImageReference, err
 			require.True(t, ok)
 			assert.Equal(t, path, ociArchRef.file, input)
 			assert.Equal(t, image.image, ociArchRef.image, input)
+			assert.Equal(t, ociArchRef.sourceIndex, image.expectedSourceIndex, input)
 		}
 	}
 
-	_, err = fn(tmpDir + ":invalid'image!value@")
-	assert.Error(t, err)
+	for _, imageSuffix := range []string{
+		":invalid'image!value@",
+		":@",
+		":@-1",
+		":@-2",
+		":@busybox",
+		":@0:buxybox",
+	} {
+		input := tmpDir + imageSuffix
+		ref, err := fn(input)
+		assert.Equal(t, ref, nil)
+		assert.Error(t, err)
+	}
 }
 
 func TestNewReference(t *testing.T) {
@@ -112,6 +131,10 @@ func TestNewReference(t *testing.T) {
 
 	_, err = NewReference(tmpDir+"/has:colon", imageValue)
 	assert.Error(t, err)
+
+	// Test private newReference
+	_, err = newReference(tmpDir, "imageName", 1, nil, nil) // Both image and sourceIndex specified
+	assert.Error(t, err)
 }
 
 // refToTempOCI creates a temporary directory and returns an reference to it.
@@ -193,7 +216,8 @@ func TestReferenceStringWithinTransport(t *testing.T) {
 
 	for _, c := range []struct{ input, result string }{
 		{"/dir1:notlatest:notlatest", "/dir1:notlatest:notlatest"}, // Explicit image
-		{"/dir3:", "/dir3:"}, // No image
+		{"/dir3:", "/dir3:"},     // No image
+		{"/dir1:@1", "/dir1:@1"}, // Explicit sourceIndex of image
 	} {
 		ref, err := ParseReference(tmpDir + c.input)
 		require.NoError(t, err, c.input)