OCI-archive multi-manifest support POC

docs/containers-transports.5.md

@@ -62,14 +62,19 @@ An image stored in the docker daemon's internal storage.
 The image must be specified as a _docker-reference_ or in an alternative _algo:digest_ format when being used as an image source.
 The _algo:digest_ refers to the image ID reported by docker-inspect(1).
 
-### **oci:**_path[:reference]_
+### **oci:**_path[:{reference|@source-index}]_
 
 An image compliant with the "Open Container Image Layout Specification" at _path_.
 Using a _reference_ is optional and allows for storing multiple images at the same _path_.
+For reading images, @_source-index_ is a zero-based index in manifest (to access untagged images).
+If neither reference nor @_source_index is specified when reading an image, the path must contain exactly one image.
 
-### **oci-archive:**_path[:reference]_
+### **oci-archive:**_path[:{reference|@source-index}]_
 
 An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.
+Using a _reference_ is optional and allows for storing multiple images at the same _path_.
+For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images).
+If neither reference nor @_source_index is specified when reading an archive, the archive must contain exactly one image.
 
 ### **ostree:**_docker-reference[@/absolute/repo/path]_
 

oci/archive/oci_dest.go

@@ -4,44 +4,67 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"os"
 
 	"github.com/containers/image/v5/internal/blobinfocache"
 	"github.com/containers/image/v5/internal/imagedestination"
 	"github.com/containers/image/v5/internal/imagedestination/impl"
 	"github.com/containers/image/v5/internal/private"
 	"github.com/containers/image/v5/internal/signature"
+	"github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
-	"github.com/containers/storage/pkg/archive"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/sirupsen/logrus"
 )
 
 type ociArchiveImageDestination struct {
 	impl.Compat
 
-	ref          ociArchiveReference
-	unpackedDest private.ImageDestination
-	tempDirRef   tempDirOCIRef
+	ref                   ociArchiveReference
+	individualWriterOrNil *Writer
+	unpackedDest          private.ImageDestination
 }
 
 // newImageDestination returns an ImageDestination for writing to an existing directory.
 func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (private.ImageDestination, error) {
-	tempDirRef, err := createOCIRef(sys, ref.image)
+	var (
+		archive, individualWriterOrNil *Writer
+		err                            error
+	)
+
+	if ref.sourceIndex != -1 {
+		return nil, fmt.Errorf("destination reference must not contain a manifest index @%d", ref.sourceIndex)
+	}
+
+	if ref.archiveWriter != nil {
+		archive = ref.archiveWriter
+		individualWriterOrNil = nil
+	} else {
+		archive, err = NewWriter(ctx, sys, ref.resolvedFile)
+		if err != nil {
+			return nil, err
+		}
+		individualWriterOrNil = archive
+	}
+	succeeded := false
+	defer func() {
+		if !succeeded && individualWriterOrNil != nil {
+			individualWriterOrNil.Close()
+		}
+	}()
+
+	layoutRef, err := layout.NewReference(archive.tempDir, ref.image)
 	if err != nil {
-		return nil, fmt.Errorf("creating oci reference: %w", err)
+		return nil, err
 	}
-	unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
+	dst, err := layoutRef.NewImageDestination(ctx, sys)
 	if err != nil {
-		if err := tempDirRef.deleteTempDir(); err != nil {
-			return nil, fmt.Errorf("deleting temp directory %q: %w", tempDirRef.tempDirectory, err)
-		}
 		return nil, err
 	}
+
+	succeeded = true
 	d := &ociArchiveImageDestination{
-		ref:          ref,
-		unpackedDest: imagedestination.FromPublic(unpackedDest),
-		tempDirRef:   tempDirRef,
+		ref:                   ref,
+		individualWriterOrNil: individualWriterOrNil,
+		unpackedDest:          imagedestination.FromPublic(dst),
 	}
 	d.Compat = impl.AddCompat(d)
 	return d, nil
@@ -53,13 +76,14 @@ func (d *ociArchiveImageDestination) Reference() types.ImageReference {
 }
 
 // Close removes resources associated with an initialized ImageDestination, if any
-// Close deletes the temp directory of the oci-archive image
 func (d *ociArchiveImageDestination) Close() error {
-	defer func() {
-		err := d.tempDirRef.deleteTempDir()
-		logrus.Debugf("Error deleting temporary directory: %v", err)
-	}()
-	return d.unpackedDest.Close()
+	if err := d.unpackedDest.Close(); err != nil {
+		return err
+	}
+	if d.individualWriterOrNil == nil {
+		return nil
+	}
+	return d.individualWriterOrNil.Close()
 }
 
 func (d *ociArchiveImageDestination) SupportedManifestMIMETypes() []string {
@@ -160,32 +184,5 @@ func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedTopleve
 	if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil {
 		return fmt.Errorf("storing image %q: %w", d.ref.image, err)
 	}
-
-	// path of directory to tar up
-	src := d.tempDirRef.tempDirectory
-	// path to save tarred up file
-	dst := d.ref.resolvedFile
-	return tarDirectory(src, dst)
-}
-
-// tar converts the directory at src and saves it to dst
-func tarDirectory(src, dst string) error {
-	// input is a stream of bytes from the archive of the directory at path
-	input, err := archive.Tar(src, archive.Uncompressed)
-	if err != nil {
-		return fmt.Errorf("retrieving stream of bytes from %q: %w", src, err)
-	}
-
-	// creates the tar file
-	outFile, err := os.Create(dst)
-	if err != nil {
-		return fmt.Errorf("creating tar file %q: %w", dst, err)
-	}
-	defer outFile.Close()
-
-	// copies the contents of the directory to the tar file
-	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
-	_, err = io.Copy(outFile, input)
-
-	return err
+	return nil
 }

oci/archive/oci_src.go

@@ -10,7 +10,7 @@ import (
 	"github.com/containers/image/v5/internal/imagesource/impl"
 	"github.com/containers/image/v5/internal/private"
 	"github.com/containers/image/v5/internal/signature"
-	ocilayout "github.com/containers/image/v5/oci/layout"
+	"github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/types"
 	digest "github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -20,30 +20,76 @@ import (
 type ociArchiveImageSource struct {
 	impl.Compat
 
-	ref         ociArchiveReference
-	unpackedSrc private.ImageSource
-	tempDirRef  tempDirOCIRef
+	ref                   ociArchiveReference
+	unpackedSrc           private.ImageSource
+	individualReaderOrNil *Reader
+}
+
+// openRef returns (layoutRef, individualReaderOrNil) for consuming ref.
+// The caller must close individualReaderOrNil (if the latter is not nil).
+func openRef(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageReference, *Reader, error) {
+	var (
+		archive, individualReaderOrNil *Reader
+		layoutRef                      types.ImageReference
+		err                            error
+	)
+
+	if ref.archiveReader != nil {
+		archive = ref.archiveReader
+		individualReaderOrNil = nil
+	} else {
+		archive, err = NewReader(ctx, sys, ref.resolvedFile)
+		if err != nil {
+			return nil, nil, err
+		}
+		individualReaderOrNil = archive
+	}
+	succeeded := false
+	defer func() {
+		if !succeeded && individualReaderOrNil != nil {
+			individualReaderOrNil.Close()
+		}
+	}()
+
+	if ref.sourceIndex != -1 {
+		layoutRef, err = layout.NewIndexReference(archive.tempDirectory, ref.sourceIndex)
+		if err != nil {
+			return nil, nil, err
+		}
+	} else {
+		layoutRef, err = layout.NewReference(archive.tempDirectory, ref.image)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	succeeded = true
+	return layoutRef, individualReaderOrNil, nil
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
-// newImageSource untars the file and saves it in a temp directory
 func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (private.ImageSource, error) {
-	tempDirRef, err := createUntarTempDir(sys, ref)
+	layoutRef, individualReaderOrNil, err := openRef(ctx, sys, ref)
 	if err != nil {
-		return nil, fmt.Errorf("creating temp directory: %w", err)
+		return nil, err
 	}
+	succeeded := false
+	defer func() {
+		if !succeeded && individualReaderOrNil != nil {
+			individualReaderOrNil.Close()
+		}
+	}()
 
-	unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
+	src, err := layoutRef.NewImageSource(ctx, sys)
 	if err != nil {
-		if err := tempDirRef.deleteTempDir(); err != nil {
-			return nil, fmt.Errorf("deleting temp directory %q: %w", tempDirRef.tempDirectory, err)
-		}
 		return nil, err
 	}
+
+	succeeded = true
 	s := &ociArchiveImageSource{
-		ref:         ref,
-		unpackedSrc: imagesource.FromPublic(unpackedSrc),
-		tempDirRef:  tempDirRef,
+		ref:                   ref,
+		unpackedSrc:           imagesource.FromPublic(src),
+		individualReaderOrNil: individualReaderOrNil,
 	}
 	s.Compat = impl.AddCompat(s)
 	return s, nil
@@ -61,16 +107,20 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im
 	if !ok {
 		return imgspecv1.Descriptor{}, errors.New("error typecasting, need type ociArchiveReference")
 	}
-	tempDirRef, err := createUntarTempDir(sys, ociArchRef)
+
+	layoutRef, individualReaderOrNil, err := openRef(context.TODO(), sys, ociArchRef)
 	if err != nil {
-		return imgspecv1.Descriptor{}, fmt.Errorf("creating temp directory: %w", err)
+		return imgspecv1.Descriptor{}, err
 	}
 	defer func() {
-		err := tempDirRef.deleteTempDir()
-		logrus.Debugf("Error deleting temporary directory: %v", err)
+		if individualReaderOrNil != nil {
+			if err := individualReaderOrNil.Close(); err != nil {
+				logrus.Debugf("Error deleting temporary directory: %v", err)
+			}
+		}
 	}()
 
-	descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
+	descriptor, err := layout.LoadManifestDescriptor(layoutRef)
 	if err != nil {
 		return imgspecv1.Descriptor{}, fmt.Errorf("loading index: %w", err)
 	}
@@ -83,13 +133,14 @@ func (s *ociArchiveImageSource) Reference() types.ImageReference {
 }
 
 // Close removes resources associated with an initialized ImageSource, if any.
-// Close deletes the temporary directory at dst
 func (s *ociArchiveImageSource) Close() error {
-	defer func() {
-		err := s.tempDirRef.deleteTempDir()
-		logrus.Debugf("error deleting tmp dir: %v", err)
-	}()
-	return s.unpackedSrc.Close()
+	if err := s.unpackedSrc.Close(); err != nil {
+		return err
+	}
+	if s.individualReaderOrNil == nil {
+		return nil
+	}
+	return s.individualReaderOrNil.Close()
 }
 
 // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).