Skip to content

Commit

Permalink
Add reader/writer for oci-archive multi image support
Browse files Browse the repository at this point in the history 
Add reader/writer with helpers to allow podman save/load multi oci-archive images.
Allow read oci-archive using source_index to point to the index from oci-archive manifest.
Also reimplement ociArchiveImage{Source,Destination} to support this.

Signed-off-by: Qi Wang <[email protected]>
Signed-off-by: Urvashi Mohnani <[email protected]>
  • Loading branch information
@QiWang19 @umohnani8
QiWang19 authored and umohnani8 committed Mar 1, 2022
1 parent d722da5 commit c4be24d
Show file tree
Hide file tree
Showing 13 changed files with 526 additions and 95 deletions.
10 changes: 6 additions & 4 deletions docs/containers-transports.5.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,16 @@ An image stored in the docker daemon's internal storage.
The image must be specified as a _docker-reference_ or in an alternative _algo:digest_ format when being used as an image source.
The _algo:digest_ refers to the image ID reported by docker-inspect(1).

### **oci:**_path[:reference]_
### **oci:**_path[:{reference|@source-index}]_

An image compliant with the "Open Container Image Layout Specification" at _path_.
Using a _reference_ is optional and allows for storing multiple images at the same _path_.
Using a @_source-index_ is optional and allows for storing multiple images at the same _path_.
For reading images, @_source-index_ is a zero-based index in manifest (to access untagged images).
If neither tag nor @_source_index is specified when reading an image, the path must contain exactly one image.

### **oci-archive:**_path[:reference]_
### **oci-archive:**_path[:{reference|@source-index}]_

An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_.
An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_. For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images). If neither tag nor @_source_index is specified when reading an archive, the archive must contain exactly one image.

### **ostree:**_docker-reference[@/absolute/repo/path]_

Expand Down
59 changes: 38 additions & 21 deletions oci/archive/oci_dest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,53 @@ package archive

import (
"context"
"fmt"
"io"
"os"

"github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/archive"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

type ociArchiveImageDestination struct {
ref ociArchiveReference
tmpDir string
unpackedDest types.ImageDestination
tempDirRef tempDirOCIRef
}

// newImageDestination returns an ImageDestination for writing to an existing directory.
func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
tempDirRef, err := createOCIRef(sys, ref.image)
var archive *Writer

if ref.sourceIndex != -1 {
return nil, fmt.Errorf("%w: destination reference must not contain a manifest index @%d", invalidOciArchiveErr, ref.sourceIndex)
}

if ref.archiveWriter != nil {
archive = ref.archiveWriter
} else {
a, err := NewWriter(ctx, sys, ref.file)
if err != nil {
return nil, err
}
archive = a
}
newref, err := layout.NewReference(archive.tempDir, ref.image)
if err != nil {
return nil, errors.Wrapf(err, "creating oci reference")
return nil, err
}
unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
dst, err := newref.NewImageDestination(ctx, sys)
if err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
}
return nil, err
}
return &ociArchiveImageDestination{ref: ref,
unpackedDest: unpackedDest,
tempDirRef: tempDirRef}, nil

return &ociArchiveImageDestination{
unpackedDest: dst,
tmpDir: archive.tempDir,
ref: ref,
}, nil
}

// Reference returns the reference used to set up this destination.
Expand All @@ -42,12 +57,10 @@ func (d *ociArchiveImageDestination) Reference() types.ImageReference {
}

// Close removes resources associated with an initialized ImageDestination, if any
// Close deletes the temp directory of the oci-archive image
func (d *ociArchiveImageDestination) Close() error {
defer func() {
err := d.tempDirRef.deleteTempDir()
logrus.Debugf("Error deleting temporary directory: %v", err)
}()
if d.ref.archiveWriter != nil {
return nil
}
return d.unpackedDest.Close()
}

Expand Down Expand Up @@ -135,11 +148,15 @@ func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatur
// after the directory is made, it is tarred up into a file and the directory is deleted
func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil {
return errors.Wrapf(err, "storing image %q", d.ref.image)
return fmt.Errorf("%w: storing image %q", err, d.ref.image)
}

if d.ref.archiveWriter != nil {
return nil
}

// path of directory to tar up
src := d.tempDirRef.tempDirectory
src := d.tmpDir
// path to save tarred up file
dst := d.ref.resolvedFile
return tarDirectory(src, dst)
Expand All @@ -150,13 +167,13 @@ func tarDirectory(src, dst string) error {
// input is a stream of bytes from the archive of the directory at path
input, err := archive.Tar(src, archive.Uncompressed)
if err != nil {
return errors.Wrapf(err, "retrieving stream of bytes from %q", src)
return fmt.Errorf("%w: retrieving stream of bytes from %q", err, src)
}

// creates the tar file
outFile, err := os.Create(dst)
if err != nil {
return errors.Wrapf(err, "creating tar file %q", dst)
return fmt.Errorf("%w: creating tar file %q", err, dst)
}
defer outFile.Close()

Expand Down
49 changes: 28 additions & 21 deletions oci/archive/oci_src.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,49 @@ package archive

import (
"context"
"fmt"
"io"

"github.com/containers/image/v5/oci/layout"
ocilayout "github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/types"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

type ociArchiveImageSource struct {
ref ociArchiveReference
unpackedSrc types.ImageSource
tempDirRef tempDirOCIRef
}

// newImageSource returns an ImageSource for reading from an existing directory.
// newImageSource untars the file and saves it in a temp directory
func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
tempDirRef, err := createUntarTempDir(sys, ref)
if err != nil {
return nil, errors.Wrap(err, "creating temp directory")
var archive *Reader

if ref.archiveReader != nil {
archive = ref.archiveReader
} else {
a, err := NewReader(ctx, sys, ref)
if err != nil {
return nil, err
}
archive = a
}

unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
newref, err := layout.NewReference(archive.tempDirectory, ref.image)
if err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
}
return nil, err
}
return &ociArchiveImageSource{ref: ref,
unpackedSrc: unpackedSrc,
tempDirRef: tempDirRef}, nil
src, err := newref.NewImageSource(ctx, sys)
if err != nil {
return nil, err
}

return &ociArchiveImageSource{
unpackedSrc: src,
ref: ref,
}, nil
}

// LoadManifestDescriptor loads the manifest
Expand All @@ -48,11 +57,11 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor,
func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
ociArchRef, ok := imgRef.(ociArchiveReference)
if !ok {
return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociArchiveReference")
return imgspecv1.Descriptor{}, fmt.Errorf("error typecasting, need type ociArchiveReference")
}
tempDirRef, err := createUntarTempDir(sys, ociArchRef)
if err != nil {
return imgspecv1.Descriptor{}, errors.Wrap(err, "creating temp directory")
return imgspecv1.Descriptor{}, fmt.Errorf("%w: creating temp directory", err)
}
defer func() {
err := tempDirRef.deleteTempDir()
Expand All @@ -61,7 +70,7 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im

descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
if err != nil {
return imgspecv1.Descriptor{}, errors.Wrap(err, "loading index")
return imgspecv1.Descriptor{}, fmt.Errorf("%w: loading index", err)
}
return descriptor, nil
}
Expand All @@ -72,12 +81,10 @@ func (s *ociArchiveImageSource) Reference() types.ImageReference {
}

// Close removes resources associated with an initialized ImageSource, if any.
// Close deletes the temporary directory at dst
func (s *ociArchiveImageSource) Close() error {
defer func() {
err := s.tempDirRef.deleteTempDir()
logrus.Debugf("error deleting tmp dir: %v", err)
}()
if s.ref.archiveReader != nil {
return nil
}
return s.unpackedSrc.Close()
}

Expand Down
75 changes: 57 additions & 18 deletions oci/archive/oci_transport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ func init() {
transports.Register(Transport)
}

var invalidOciArchiveErr error = errors.New("Invalid oci-archive: reference")

// Transport is an ImageTransport for OCI archive
// it creates an oci-archive tar file by calling into the OCI transport
// tarring the directory created by oci and deleting the directory
Expand All @@ -32,9 +34,12 @@ type ociArchiveTransport struct{}

// ociArchiveReference is an ImageReference for OCI Archive paths
type ociArchiveReference struct {
file string
resolvedFile string
image string
file string
resolvedFile string
image string
sourceIndex int
archiveReader *Reader
archiveWriter *Writer
}

func (t ociArchiveTransport) Name() string {
Expand All @@ -54,12 +59,24 @@ func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) erro

// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference.
func ParseReference(reference string) (types.ImageReference, error) {
file, image := internal.SplitPathAndImage(reference)
return NewReference(file, image)
file, image, index, err := internal.ParseReferenceIntoElements(reference)
if err != nil {
return nil, err
}
return newReference(file, image, index, nil, nil)
}

// NewReference returns an OCI reference for a file and a image.
// NewReference returns an OCI reference for a file and an image.
func NewReference(file, image string) (types.ImageReference, error) {
return newReference(file, image, -1, nil, nil)
}

// NewIndexReference returns an OCI reference for a file and a zero-based source manifest index.
func NewIndexReference(file string, sourceIndex int) (types.ImageReference, error) {
return newReference(file, "", sourceIndex, nil, nil)
}

func newReference(file, image string, sourceIndex int, archiveReader *Reader, archiveWriter *Writer) (types.ImageReference, error) {
resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file)
if err != nil {
return nil, err
Expand All @@ -73,7 +90,20 @@ func NewReference(file, image string) (types.ImageReference, error) {
return nil, err
}

return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil
if sourceIndex != -1 && sourceIndex < 0 {
return nil, fmt.Errorf("%w: index @%d must not be negative", invalidOciArchiveErr, sourceIndex)
}
if sourceIndex != -1 && image != "" {
return nil, fmt.Errorf("%w: cannot set image %s and index @%d at the same time", invalidOciArchiveErr, image, sourceIndex)
}
return ociArchiveReference{
file: file,
resolvedFile: resolved,
image: image,
sourceIndex: sourceIndex,
archiveReader: archiveReader,
archiveWriter: archiveWriter,
}, nil
}

func (ref ociArchiveReference) Transport() types.ImageTransport {
Expand All @@ -83,7 +113,10 @@ func (ref ociArchiveReference) Transport() types.ImageTransport {
// StringWithinTransport returns a string representation of the reference, which MUST be such that
// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
func (ref ociArchiveReference) StringWithinTransport() string {
return fmt.Sprintf("%s:%s", ref.file, ref.image)
if ref.sourceIndex == -1 {
return fmt.Sprintf("%s:%s", ref.file, ref.image)
}
return fmt.Sprintf("%s:@%d", ref.file, ref.sourceIndex)
}

// DockerReference returns a Docker reference associated with this reference
Expand Down Expand Up @@ -144,7 +177,7 @@ func (ref ociArchiveReference) NewImageDestination(ctx context.Context, sys *typ

// DeleteImage deletes the named image from the registry, if supported.
func (ref ociArchiveReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
return errors.Errorf("Deleting images not implemented for oci: images")
return fmt.Errorf("Deleting images not implemented for oci: images")
}

// struct to store the ociReference and temporary directory returned by createOCIRef
Expand All @@ -160,14 +193,20 @@ func (t *tempDirOCIRef) deleteTempDir() error {

// createOCIRef creates the oci reference of the image
// If SystemContext.BigFilesTemporaryDir not "", overrides the temporary directory to use for storing big files
func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) {
func createOCIRef(sys *types.SystemContext, image string, sourceIndex int) (tempDirOCIRef, error) {
dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci")
if err != nil {
return tempDirOCIRef{}, errors.Wrapf(err, "creating temp directory")
return tempDirOCIRef{}, fmt.Errorf("%w: creating temp directory", err)
}
ociRef, err := ocilayout.NewReference(dir, image)
if err != nil {
return tempDirOCIRef{}, err
var ociRef types.ImageReference
if sourceIndex > -1 {
if ociRef, err = ocilayout.NewIndexReference(dir, sourceIndex); err != nil {
return tempDirOCIRef{}, err
}
} else {
if ociRef, err = ocilayout.NewReference(dir, image); err != nil {
return tempDirOCIRef{}, err
}
}

tempDirRef := tempDirOCIRef{tempDirectory: dir, ociRefExtracted: ociRef}
Expand All @@ -176,9 +215,9 @@ func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error)

// creates the temporary directory and copies the tarred content to it
func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (tempDirOCIRef, error) {
tempDirRef, err := createOCIRef(sys, ref.image)
tempDirRef, err := createOCIRef(sys, ref.image, ref.sourceIndex)
if err != nil {
return tempDirOCIRef{}, errors.Wrap(err, "creating oci reference")
return tempDirOCIRef{}, fmt.Errorf("%w: creating oci reference", err)
}
src := ref.resolvedFile
dst := tempDirRef.tempDirectory
Expand All @@ -190,9 +229,9 @@ func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (temp
defer arch.Close()
if err := archive.NewDefaultArchiver().Untar(arch, dst, &archive.TarOptions{NoLchown: true}); err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return tempDirOCIRef{}, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
return tempDirOCIRef{}, fmt.Errorf("%w: deleting temp directory %q", err, tempDirRef.tempDirectory)
}
return tempDirOCIRef{}, errors.Wrapf(err, "untarring file %q", tempDirRef.tempDirectory)
return tempDirOCIRef{}, fmt.Errorf("%w: untarring file %q", err, tempDirRef.tempDirectory)
}
return tempDirRef, nil
}
Loading

0 comments on commit c4be24d

Please sign in to comment.