Skip to content

Commit

Permalink
Add reader/writer for oci-archive multi image support
Browse files Browse the repository at this point in the history 
Add reader/writer with helpers to allow podman save/load multi oci-archive images.
Allow read oci-archive using source_index to point to the index from oci-archive manifest.
Also reimplement ociArchiveImage{Source,Destination} to support this.

Signed-off-by: Qi Wang <[email protected]>
Signed-off-by: Urvashi Mohnani <[email protected]>
  • Loading branch information
@QiWang19 @umohnani8
QiWang19 authored and umohnani8 committed Mar 9, 2022
1 parent e6b4dfb commit 2de595a
Show file tree
Hide file tree
Showing 14 changed files with 826 additions and 162 deletions.
11 changes: 8 additions & 3 deletions docs/containers-transports.5.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,19 @@ An image stored in the docker daemon's internal storage.
The image must be specified as a _docker-reference_ or in an alternative _algo:digest_ format when being used as an image source.
The _algo:digest_ refers to the image ID reported by docker-inspect(1).

### **oci:**_path[:reference]_
### **oci:**_path[:{reference|@source-index}]_

An image compliant with the "Open Container Image Layout Specification" at _path_.
Using a _reference_ is optional and allows for storing multiple images at the same _path_.
For reading images, @_source-index_ is a zero-based index in manifest (to access untagged images).
If neither reference nor @_source_index is specified when reading an image, the path must contain exactly one image.

### **oci-archive:**_path[:reference]_
### **oci-archive:**_path[:{reference|@source-index}]_

An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.
An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.
Using a _reference_ is optional and allows for storing multiple images at the same _path_.
For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images).
If neither reference nor @_source_index is specified when reading an archive, the archive must contain exactly one image.

### **ostree:**_docker-reference[@/absolute/repo/path]_

Expand Down
93 changes: 42 additions & 51 deletions oci/archive/oci_dest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,57 @@ package archive

import (
"context"
"fmt"
"io"
"os"

"github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/archive"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

type ociArchiveImageDestination struct {
ref ociArchiveReference
unpackedDest types.ImageDestination
tempDirRef tempDirOCIRef
ref ociArchiveReference
individualWriterOrNil *Writer
unpackedDest types.ImageDestination
}

// newImageDestination returns an ImageDestination for writing to an existing directory.
func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
tempDirRef, err := createOCIRef(sys, ref.image)
var (
archive, individualWriterOrNil *Writer
err error
)

if ref.sourceIndex != -1 {
return nil, fmt.Errorf("destination reference must not contain a manifest index @%d: %w", ref.sourceIndex, invalidOciArchiveErr)
}

if ref.archiveWriter != nil {
archive = ref.archiveWriter
individualWriterOrNil = nil
} else {
archive, err = NewWriter(ctx, sys, ref.file)
if err != nil {
return nil, err
}
individualWriterOrNil = archive
}
newref, err := layout.NewReference(archive.tempDir, ref.image)
if err != nil {
return nil, errors.Wrapf(err, "creating oci reference")
archive.Close()
return nil, err
}
unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
dst, err := newref.NewImageDestination(ctx, sys)
if err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
}
archive.Close()
return nil, err
}
return &ociArchiveImageDestination{ref: ref,
unpackedDest: unpackedDest,
tempDirRef: tempDirRef}, nil

return &ociArchiveImageDestination{
unpackedDest: dst,
individualWriterOrNil: individualWriterOrNil,
ref: ref,
}, nil
}

// Reference returns the reference used to set up this destination.
Expand All @@ -42,13 +61,12 @@ func (d *ociArchiveImageDestination) Reference() types.ImageReference {
}

// Close removes resources associated with an initialized ImageDestination, if any
// Close deletes the temp directory of the oci-archive image
func (d *ociArchiveImageDestination) Close() error {
defer func() {
err := d.tempDirRef.deleteTempDir()
logrus.Debugf("Error deleting temporary directory: %v", err)
}()
return d.unpackedDest.Close()
defer d.unpackedDest.Close()
if d.ref.archiveWriter != nil || d.individualWriterOrNil == nil {
return nil
}
return d.individualWriterOrNil.Close()
}

func (d *ociArchiveImageDestination) SupportedManifestMIMETypes() []string {
Expand Down Expand Up @@ -135,34 +153,7 @@ func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatur
// after the directory is made, it is tarred up into a file and the directory is deleted
func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil {
return errors.Wrapf(err, "storing image %q", d.ref.image)
return fmt.Errorf("storing image %q: %w", d.ref.image, err)
}

// path of directory to tar up
src := d.tempDirRef.tempDirectory
// path to save tarred up file
dst := d.ref.resolvedFile
return tarDirectory(src, dst)
}

// tar converts the directory at src and saves it to dst
func tarDirectory(src, dst string) error {
// input is a stream of bytes from the archive of the directory at path
input, err := archive.Tar(src, archive.Uncompressed)
if err != nil {
return errors.Wrapf(err, "retrieving stream of bytes from %q", src)
}

// creates the tar file
outFile, err := os.Create(dst)
if err != nil {
return errors.Wrapf(err, "creating tar file %q", dst)
}
defer outFile.Close()

// copies the contents of the directory to the tar file
// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
_, err = io.Copy(outFile, input)

return err
return nil
}
74 changes: 50 additions & 24 deletions oci/archive/oci_src.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,67 @@ package archive

import (
"context"
"fmt"
"io"

"github.com/containers/image/v5/oci/layout"
ocilayout "github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/types"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

type ociArchiveImageSource struct {
ref ociArchiveReference
unpackedSrc types.ImageSource
tempDirRef tempDirOCIRef
ref ociArchiveReference
unpackedSrc types.ImageSource
individualReaderOrNil *Reader
}

// newImageSource returns an ImageSource for reading from an existing directory.
// newImageSource untars the file and saves it in a temp directory
func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
tempDirRef, err := createUntarTempDir(sys, ref)
if err != nil {
return nil, errors.Wrap(err, "creating temp directory")
var (
archive, individualReaderOrNil *Reader
newref types.ImageReference
err error
)

if ref.archiveReader != nil {
archive = ref.archiveReader
individualReaderOrNil = nil
} else {
archive, err = NewReader(ctx, sys, ref)
if err != nil {
return nil, err
}
individualReaderOrNil = archive
}

unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
if err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
if ref.sourceIndex != -1 {
newref, err = layout.NewIndexReference(archive.tempDirectory, ref.sourceIndex)
if err != nil {
archive.Close()
return nil, err
}
} else {
newref, err = layout.NewReference(archive.tempDirectory, ref.image)
if err != nil {
archive.Close()
return nil, err
}
}

src, err := newref.NewImageSource(ctx, sys)
if err != nil {
archive.Close()
return nil, err
}
return &ociArchiveImageSource{ref: ref,
unpackedSrc: unpackedSrc,
tempDirRef: tempDirRef}, nil

return &ociArchiveImageSource{
unpackedSrc: src,
ref: ref,
individualReaderOrNil: individualReaderOrNil,
}, nil
}

// LoadManifestDescriptor loads the manifest
Expand All @@ -48,11 +75,11 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor,
func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
ociArchRef, ok := imgRef.(ociArchiveReference)
if !ok {
return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociArchiveReference")
return imgspecv1.Descriptor{}, fmt.Errorf("error typecasting, need type ociArchiveReference")
}
tempDirRef, err := createUntarTempDir(sys, ociArchRef)
if err != nil {
return imgspecv1.Descriptor{}, errors.Wrap(err, "creating temp directory")
return imgspecv1.Descriptor{}, fmt.Errorf("creating temp directory: %w", err)
}
defer func() {
err := tempDirRef.deleteTempDir()
Expand All @@ -61,7 +88,7 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im

descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
if err != nil {
return imgspecv1.Descriptor{}, errors.Wrap(err, "loading index")
return imgspecv1.Descriptor{}, fmt.Errorf("loading index: %w", err)
}
return descriptor, nil
}
Expand All @@ -72,13 +99,12 @@ func (s *ociArchiveImageSource) Reference() types.ImageReference {
}

// Close removes resources associated with an initialized ImageSource, if any.
// Close deletes the temporary directory at dst
func (s *ociArchiveImageSource) Close() error {
defer func() {
err := s.tempDirRef.deleteTempDir()
logrus.Debugf("error deleting tmp dir: %v", err)
}()
return s.unpackedSrc.Close()
defer s.unpackedSrc.Close()
if s.ref.archiveReader != nil || s.individualReaderOrNil == nil {
return nil
}
return s.individualReaderOrNil.Close()
}

// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
Expand Down
Loading

0 comments on commit 2de595a

Please sign in to comment.