diff --git a/docs/containers-transports.5.md b/docs/containers-transports.5.md index 6c94d0a1fa..30571c67cd 100644 --- a/docs/containers-transports.5.md +++ b/docs/containers-transports.5.md @@ -60,11 +60,13 @@ The _algo:digest_ refers to the image ID reported by docker-inspect(1). ### **oci:**_path[:reference]_ An image compliant with the "Open Container Image Layout Specification" at _path_. -Using a _reference_ is optional and allows for storing multiple images at the same _path_. +Using a @_source-index_ is optional and allows for storing multiple images at the same _path_. +For reading images, @_source-index_ is a zero-based index in manifest (to access untagged images). +If neither tag nor @_source_index is specified when reading an image, the path must contain exactly one image. ### **oci-archive:**_path[:reference]_ -An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_. +An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_. For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images). If neither tag nor @_source_index is specified when reading an archive, the archive must contain exactly one image. ### **ostree:**_docker-reference[@/absolute/repo/path]_ diff --git a/oci/archive/oci_dest.go b/oci/archive/oci_dest.go index 3d8738db53..7447350e05 100644 --- a/oci/archive/oci_dest.go +++ b/oci/archive/oci_dest.go @@ -2,38 +2,52 @@ package archive import ( "context" + "fmt" "io" "os" + "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/archive" digest "github.com/opencontainers/go-digest" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) type ociArchiveImageDestination struct { - ref ociArchiveReference - unpackedDest types.ImageDestination - tempDirRef tempDirOCIRef + ref ociArchiveReference + types.ImageDestination } // newImageDestination returns an ImageDestination for writing to an existing directory. func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) { - tempDirRef, err := createOCIRef(sys, ref.image) + var archive *Writer + + if ref.sourceIndex != -1 { + return nil, fmt.Errorf("%w: destination reference must not contain a manifest index @%d", invalidOciArchiveErr, ref.sourceIndex) + } + + if ref.archiveWriter != nil { + archive = ref.archiveWriter + } else { + a, err := NewWriter(ctx, sys, ref.file) + if err != nil { + return nil, err + } + defer a.Close() + archive = a + } + newref, err := layout.NewReference(archive.tempDir, ref.image) if err != nil { - return nil, errors.Wrapf(err, "creating oci reference") + return nil, err } - unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys) + dst, err := newref.NewImageDestination(ctx, sys) if err != nil { - if err := tempDirRef.deleteTempDir(); err != nil { - return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory) - } return nil, err } - return &ociArchiveImageDestination{ref: ref, - unpackedDest: unpackedDest, - tempDirRef: tempDirRef}, nil + + return &ociArchiveImageDestination{ + ImageDestination: dst, + ref: ref, + }, nil } // Reference returns the reference used to set up this destination. @@ -44,42 +58,41 @@ func (d *ociArchiveImageDestination) Reference() types.ImageReference { // Close removes resources associated with an initialized ImageDestination, if any // Close deletes the temp directory of the oci-archive image func (d *ociArchiveImageDestination) Close() error { - defer func() { - err := d.tempDirRef.deleteTempDir() - logrus.Debugf("Error deleting temporary directory: %v", err) - }() - return d.unpackedDest.Close() + if d.ref.archiveWriter != nil { + return nil + } + return d.ImageDestination.Close() } func (d *ociArchiveImageDestination) SupportedManifestMIMETypes() []string { - return d.unpackedDest.SupportedManifestMIMETypes() + return d.ImageDestination.SupportedManifestMIMETypes() } // SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures func (d *ociArchiveImageDestination) SupportsSignatures(ctx context.Context) error { - return d.unpackedDest.SupportsSignatures(ctx) + return d.ImageDestination.SupportsSignatures(ctx) } func (d *ociArchiveImageDestination) DesiredLayerCompression() types.LayerCompression { - return d.unpackedDest.DesiredLayerCompression() + return d.ImageDestination.DesiredLayerCompression() } // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually // uploaded to the image destination, true otherwise. func (d *ociArchiveImageDestination) AcceptsForeignLayerURLs() bool { - return d.unpackedDest.AcceptsForeignLayerURLs() + return d.ImageDestination.AcceptsForeignLayerURLs() } // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise func (d *ociArchiveImageDestination) MustMatchRuntimeOS() bool { - return d.unpackedDest.MustMatchRuntimeOS() + return d.ImageDestination.MustMatchRuntimeOS() } // IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), // and would prefer to receive an unmodified manifest instead of one modified for the destination. // Does not make a difference if Reference().DockerReference() is nil. func (d *ociArchiveImageDestination) IgnoresEmbeddedDockerReference() bool { - return d.unpackedDest.IgnoresEmbeddedDockerReference() + return d.ImageDestination.IgnoresEmbeddedDockerReference() } // HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. @@ -96,7 +109,7 @@ func (d *ociArchiveImageDestination) HasThreadSafePutBlob() bool { // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. func (d *ociArchiveImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { - return d.unpackedDest.PutBlob(ctx, stream, inputInfo, cache, isConfig) + return d.ImageDestination.PutBlob(ctx, stream, inputInfo, cache, isConfig) } // TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination @@ -109,7 +122,7 @@ func (d *ociArchiveImageDestination) PutBlob(ctx context.Context, stream io.Read // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. // May use and/or update cache. func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { - return d.unpackedDest.TryReusingBlob(ctx, info, cache, canSubstitute) + return d.ImageDestination.TryReusingBlob(ctx, info, cache, canSubstitute) } // PutManifest writes the manifest to the destination. @@ -118,14 +131,14 @@ func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info ty // It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated // by `manifest.Digest()`. func (d *ociArchiveImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { - return d.unpackedDest.PutManifest(ctx, m, instanceDigest) + return d.ImageDestination.PutManifest(ctx, m, instanceDigest) } // PutSignatures writes a set of signatures to the destination. // If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for // (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { - return d.unpackedDest.PutSignatures(ctx, signatures, instanceDigest) + return d.ImageDestination.PutSignatures(ctx, signatures, instanceDigest) } // Commit marks the process of storing the image as successful and asks for the image to be persisted @@ -134,12 +147,16 @@ func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatur // original manifest list digest, if desired. // after the directory is made, it is tarred up into a file and the directory is deleted func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { - if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil { - return errors.Wrapf(err, "storing image %q", d.ref.image) + if err := d.ImageDestination.Commit(ctx, unparsedToplevel); err != nil { + return fmt.Errorf("%w: storing image %q", err, d.ref.image) + } + + if d.ref.archiveWriter != nil { + return nil } // path of directory to tar up - src := d.tempDirRef.tempDirectory + src := d.ref.archiveReader.tempDirectory // path to save tarred up file dst := d.ref.resolvedFile return tarDirectory(src, dst) @@ -150,13 +167,13 @@ func tarDirectory(src, dst string) error { // input is a stream of bytes from the archive of the directory at path input, err := archive.Tar(src, archive.Uncompressed) if err != nil { - return errors.Wrapf(err, "retrieving stream of bytes from %q", src) + return fmt.Errorf("%w: retrieving stream of bytes from %q", err, src) } // creates the tar file outFile, err := os.Create(dst) if err != nil { - return errors.Wrapf(err, "creating tar file %q", dst) + return fmt.Errorf("%w: creating tar file %q", err, dst) } defer outFile.Close() diff --git a/oci/archive/oci_src.go b/oci/archive/oci_src.go index 20b392dc0e..536131a249 100644 --- a/oci/archive/oci_src.go +++ b/oci/archive/oci_src.go @@ -2,40 +2,51 @@ package archive import ( "context" + "fmt" "io" + "github.com/containers/image/v5/oci/layout" ocilayout "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/types" digest "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" ) type ociArchiveImageSource struct { - ref ociArchiveReference - unpackedSrc types.ImageSource - tempDirRef tempDirOCIRef + ref ociArchiveReference + types.ImageSource } // newImageSource returns an ImageSource for reading from an existing directory. -// newImageSource untars the file and saves it in a temp directory func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) { - tempDirRef, err := createUntarTempDir(sys, ref) - if err != nil { - return nil, errors.Wrap(err, "creating temp directory") + var archive *Reader + + if ref.archiveReader != nil { + archive = ref.archiveReader + ref.closeArchive = false + } else { + a, err := NewReader(ctx, sys, ref) + if err != nil { + return nil, err + } + archive = a + ref.closeArchive = true } - unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys) + newref, err := layout.NewReference(archive.tempDirectory, ref.image) if err != nil { - if err := tempDirRef.deleteTempDir(); err != nil { - return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory) - } return nil, err } - return &ociArchiveImageSource{ref: ref, - unpackedSrc: unpackedSrc, - tempDirRef: tempDirRef}, nil + src, err := newref.NewImageSource(ctx, sys) + if err != nil { + return nil, err + } + + return &ociArchiveImageSource{ + ImageSource: src, + ref: ref, + }, nil } // LoadManifestDescriptor loads the manifest @@ -48,11 +59,11 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor, func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.ImageReference) (imgspecv1.Descriptor, error) { ociArchRef, ok := imgRef.(ociArchiveReference) if !ok { - return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociArchiveReference") + return imgspecv1.Descriptor{}, fmt.Errorf("error typecasting, need type ociArchiveReference") } tempDirRef, err := createUntarTempDir(sys, ociArchRef) if err != nil { - return imgspecv1.Descriptor{}, errors.Wrap(err, "creating temp directory") + return imgspecv1.Descriptor{}, fmt.Errorf("%w: creating temp directory", err) } defer func() { err := tempDirRef.deleteTempDir() @@ -61,7 +72,7 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted) if err != nil { - return imgspecv1.Descriptor{}, errors.Wrap(err, "loading index") + return imgspecv1.Descriptor{}, fmt.Errorf("%w: loading index", err) } return descriptor, nil } @@ -72,13 +83,11 @@ func (s *ociArchiveImageSource) Reference() types.ImageReference { } // Close removes resources associated with an initialized ImageSource, if any. -// Close deletes the temporary directory at dst func (s *ociArchiveImageSource) Close() error { - defer func() { - err := s.tempDirRef.deleteTempDir() - logrus.Debugf("error deleting tmp dir: %v", err) - }() - return s.unpackedSrc.Close() + if s.ref.archiveReader != nil { + return nil + } + return s.ImageSource.Close() } // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). @@ -86,7 +95,7 @@ func (s *ociArchiveImageSource) Close() error { // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). func (s *ociArchiveImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { - return s.unpackedSrc.GetManifest(ctx, instanceDigest) + return s.ImageSource.GetManifest(ctx, instanceDigest) } // HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. @@ -98,7 +107,7 @@ func (s *ociArchiveImageSource) HasThreadSafeGetBlob() bool { // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. func (s *ociArchiveImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) { - return s.unpackedSrc.GetBlob(ctx, info, cache) + return s.ImageSource.GetBlob(ctx, info, cache) } // GetSignatures returns the image's signatures. It may use a remote (= slow) service. @@ -106,7 +115,7 @@ func (s *ociArchiveImageSource) GetBlob(ctx context.Context, info types.BlobInfo // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list // (e.g. if the source never returns manifest lists). func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - return s.unpackedSrc.GetSignatures(ctx, instanceDigest) + return s.ImageSource.GetSignatures(ctx, instanceDigest) } // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer @@ -118,5 +127,5 @@ func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDiges // The Digest field is guaranteed to be provided; Size may be -1. // WARNING: The list may contain duplicates, and they are semantically relevant. func (s *ociArchiveImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { - return s.unpackedSrc.LayerInfosForCopy(ctx, instanceDigest) + return s.ImageSource.LayerInfosForCopy(ctx, instanceDigest) } diff --git a/oci/archive/oci_transport.go b/oci/archive/oci_transport.go index 54d325d34d..c6a68c6e88 100644 --- a/oci/archive/oci_transport.go +++ b/oci/archive/oci_transport.go @@ -23,6 +23,8 @@ func init() { transports.Register(Transport) } +var invalidOciArchiveErr error = errors.New("Invalid oci archive: reference") + // Transport is an ImageTransport for OCI archive // it creates an oci-archive tar file by calling into the OCI transport // tarring the directory created by oci and deleting the directory @@ -32,9 +34,13 @@ type ociArchiveTransport struct{} // ociArchiveReference is an ImageReference for OCI Archive paths type ociArchiveReference struct { - file string - resolvedFile string - image string + file string + resolvedFile string + image string + sourceIndex int + closeArchive bool + archiveReader *Reader + archiveWriter *Writer } func (t ociArchiveTransport) Name() string { @@ -54,12 +60,24 @@ func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) erro // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { - file, image := internal.SplitPathAndImage(reference) - return NewReference(file, image) + file, image, index, err := internal.ParseReferenceIntoElements(reference) + if err != nil { + return nil, err + } + return newReference(file, image, index, nil, nil) } -// NewReference returns an OCI reference for a file and a image. +// NewReference returns an OCI reference for a file and an image. func NewReference(file, image string) (types.ImageReference, error) { + return newReference(file, image, -1, nil, nil) +} + +// NewIndexReference returns an OCI reference for a file and sourceIndex points to the n-th image in the manifest. +func NewIndexReference(file string, sourceIndex int) (types.ImageReference, error) { + return newReference(file, "", sourceIndex, nil, nil) +} + +func newReference(file, image string, sourceIndex int, archiveReader *Reader, archiveWriter *Writer) (types.ImageReference, error) { resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file) if err != nil { return nil, err @@ -73,7 +91,20 @@ func NewReference(file, image string) (types.ImageReference, error) { return nil, err } - return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil + if sourceIndex != -1 && sourceIndex < 0 { + return nil, fmt.Errorf("%w: index @%d must not be negative", invalidOciArchiveErr, sourceIndex) + } + if sourceIndex != -1 && image != "" { + return nil, fmt.Errorf("%w: cannot set image %s and index @%d at the same time", invalidOciArchiveErr, image, sourceIndex) + } + return ociArchiveReference{ + file: file, + resolvedFile: resolved, + image: image, + sourceIndex: sourceIndex, + archiveReader: archiveReader, + archiveWriter: archiveWriter, + }, nil } func (ref ociArchiveReference) Transport() types.ImageTransport { @@ -83,7 +114,10 @@ func (ref ociArchiveReference) Transport() types.ImageTransport { // StringWithinTransport returns a string representation of the reference, which MUST be such that // reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference. func (ref ociArchiveReference) StringWithinTransport() string { - return fmt.Sprintf("%s:%s", ref.file, ref.image) + if ref.sourceIndex == -1 { + return fmt.Sprintf("%s:%s", ref.file, ref.image) + } + return fmt.Sprintf("%s:@%d", ref.file, ref.sourceIndex) } // DockerReference returns a Docker reference associated with this reference @@ -144,7 +178,7 @@ func (ref ociArchiveReference) NewImageDestination(ctx context.Context, sys *typ // DeleteImage deletes the named image from the registry, if supported. func (ref ociArchiveReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { - return errors.Errorf("Deleting images not implemented for oci: images") + return fmt.Errorf("Deleting images not implemented for oci: images") } // struct to store the ociReference and temporary directory returned by createOCIRef @@ -160,14 +194,20 @@ func (t *tempDirOCIRef) deleteTempDir() error { // createOCIRef creates the oci reference of the image // If SystemContext.BigFilesTemporaryDir not "", overrides the temporary directory to use for storing big files -func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) { +func createOCIRef(sys *types.SystemContext, image string, sourceIndex int) (tempDirOCIRef, error) { dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci") if err != nil { - return tempDirOCIRef{}, errors.Wrapf(err, "creating temp directory") + return tempDirOCIRef{}, fmt.Errorf("%w: creating temp directory", err) } - ociRef, err := ocilayout.NewReference(dir, image) - if err != nil { - return tempDirOCIRef{}, err + var ociRef types.ImageReference + if sourceIndex > -1 { + if ociRef, err = ocilayout.NewIndexReference(dir, sourceIndex); err != nil { + return tempDirOCIRef{}, err + } + } else { + if ociRef, err = ocilayout.NewReference(dir, image); err != nil { + return tempDirOCIRef{}, err + } } tempDirRef := tempDirOCIRef{tempDirectory: dir, ociRefExtracted: ociRef} @@ -176,9 +216,9 @@ func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) // creates the temporary directory and copies the tarred content to it func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (tempDirOCIRef, error) { - tempDirRef, err := createOCIRef(sys, ref.image) + tempDirRef, err := createOCIRef(sys, ref.image, ref.sourceIndex) if err != nil { - return tempDirOCIRef{}, errors.Wrap(err, "creating oci reference") + return tempDirOCIRef{}, fmt.Errorf("%w: creating oci reference", err) } src := ref.resolvedFile dst := tempDirRef.tempDirectory @@ -190,9 +230,9 @@ func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (temp defer arch.Close() if err := archive.NewDefaultArchiver().Untar(arch, dst, &archive.TarOptions{NoLchown: true}); err != nil { if err := tempDirRef.deleteTempDir(); err != nil { - return tempDirOCIRef{}, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory) + return tempDirOCIRef{}, fmt.Errorf("%w: deleting temp directory %q", err, tempDirRef.tempDirectory) } - return tempDirOCIRef{}, errors.Wrapf(err, "untarring file %q", tempDirRef.tempDirectory) + return tempDirOCIRef{}, fmt.Errorf("%w: untarring file %q", err, tempDirRef.tempDirectory) } return tempDirRef, nil } diff --git a/oci/archive/oci_transport_test.go b/oci/archive/oci_transport_test.go index 046e2c2b9c..6040f7bc24 100644 --- a/oci/archive/oci_transport_test.go +++ b/oci/archive/oci_transport_test.go @@ -60,11 +60,18 @@ func testParseReference(t *testing.T, fn func(string) (types.ImageReference, err "relativepath", tmpDir + "/thisdoesnotexist", } { - for _, image := range []struct{ suffix, image string }{ - {":notlatest:image", "notlatest:image"}, - {":latestimage", "latestimage"}, - {":", ""}, - {"", ""}, + for _, image := range []struct { + suffix, image string + expectedSourceIndex int + }{ + {":notlatest:image", "notlatest:image", -1}, + {":latestimage", "latestimage", -1}, + {":busybox@0", "busybox@0", -1}, + {":", "", -1}, // No Image + {"", "", -1}, + {":@0", "", 0}, // Explicit sourceIndex of image + {":@10", "", 10}, + {":@999999", "", 999999}, } { input := path + image.suffix ref, err := fn(input) @@ -73,11 +80,23 @@ func testParseReference(t *testing.T, fn func(string) (types.ImageReference, err require.True(t, ok) assert.Equal(t, path, ociArchRef.file, input) assert.Equal(t, image.image, ociArchRef.image, input) + assert.Equal(t, ociArchRef.sourceIndex, image.expectedSourceIndex, input) } } - _, err = fn(tmpDir + ":invalid'image!value@") - assert.Error(t, err) + for _, imageSuffix := range []string{ + ":invalid'image!value@", + ":@", + ":@-1", + ":@-2", + ":@busybox", + ":@0:buxybox", + } { + input := tmpDir + imageSuffix + ref, err := fn(input) + assert.Equal(t, ref, nil) + assert.Error(t, err) + } } func TestNewReference(t *testing.T) { @@ -112,6 +131,10 @@ func TestNewReference(t *testing.T) { _, err = NewReference(tmpDir+"/has:colon", imageValue) assert.Error(t, err) + + // Test private newReference + _, err = newReference(tmpDir, "imageName", 1, nil, nil) // Both image and sourceIndex specified + assert.Error(t, err) } // refToTempOCI creates a temporary directory and returns an reference to it. @@ -193,7 +216,8 @@ func TestReferenceStringWithinTransport(t *testing.T) { for _, c := range []struct{ input, result string }{ {"/dir1:notlatest:notlatest", "/dir1:notlatest:notlatest"}, // Explicit image - {"/dir3:", "/dir3:"}, // No image + {"/dir3:", "/dir3:"}, // No image + {"/dir1:@1", "/dir1:@1"}, // Explicit sourceIndex of image } { ref, err := ParseReference(tmpDir + c.input) require.NoError(t, err, c.input) diff --git a/oci/archive/reader.go b/oci/archive/reader.go new file mode 100644 index 0000000000..48d7205bf4 --- /dev/null +++ b/oci/archive/reader.go @@ -0,0 +1,125 @@ +package archive + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/oci/internal" + "github.com/containers/image/v5/oci/layout" + "github.com/containers/image/v5/transports" + "github.com/containers/image/v5/types" + "github.com/containers/storage/pkg/archive" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// Reader manages the temp directory that the oci archive is untarred to and the +// manifest of the images. It allows listing its contents and accessing +// individual images with less overhead than creating image references individually +// (because the archive is, if necessary, copied or decompressed only once) +type Reader struct { + manifest *imgspecv1.Index + tempDirectory string + path string // The original, user-specified path; not the maintained temporary file, if any +} + +// NewReader returns a Reader for src. The caller should call Close() on the returned object +func NewReader(ctx context.Context, sys *types.SystemContext, ref types.ImageReference) (*Reader, error) { + standalone, ok := ref.(ociArchiveReference) + if !ok { + return nil, errors.Errorf("Internal error: NewReader called for a non-oci/archive ImageReference %s", transports.ImageName(ref)) + } + if standalone.archiveReader != nil { + return nil, errors.Errorf("Internal error: NewReader called for a reader-bound reference %s", standalone.StringWithinTransport()) + } + + src := standalone.resolvedFile + arch, err := os.Open(src) + if err != nil { + return nil, err + } + defer arch.Close() + + dst, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci") + if err != nil { + return nil, errors.Wrap(err, "error creating temp directory") + } + + reader := Reader{ + tempDirectory: dst, + path: src, + } + + succeeded := false + defer func() { + if !succeeded { + reader.Close() + } + }() + if err := archive.NewDefaultArchiver().Untar(arch, dst, &archive.TarOptions{NoLchown: true}); err != nil { + return nil, errors.Wrapf(err, "error untarring file %q", dst) + } + + indexJSON, err := os.Open(filepath.Join(dst, "index.json")) + if err != nil { + return nil, err + } + defer indexJSON.Close() + reader.manifest = &imgspecv1.Index{} + if err := json.NewDecoder(indexJSON).Decode(reader.manifest); err != nil { + return nil, err + } + succeeded = true + return &reader, nil +} + +// ListResult wraps the image reference and the manifest for loading +type ListResult struct { + ImageRef types.ImageReference + ManifestDescriptor imgspecv1.Descriptor +} + +// List returns a list of Reference for images in the reader +// the ImageReferences are valid only until the Reader is closed. +func (r *Reader) List() ([]ListResult, error) { + var res []ListResult + + for i, md := range r.manifest.Manifests { + var ( + ref types.ImageReference + err error + ) + refName := internal.NameFromAnnotations(md.Annotations) + if refName != "" { + if ref, err = layout.NewReference(r.tempDirectory, refName); err != nil { + return nil, err + } + } else { + if ref, err = layout.NewIndexReference(r.tempDirectory, i); err != nil { + return nil, err + } + } + // if didn't get ref from the above then we fall to this + if ref == nil { + ref, err = newReference(r.path, "", -1, r, nil) + if err != nil { + return nil, errors.Errorf("error creating image reference: %v", err) + } + } + reference := ListResult{ + ImageRef: ref, + ManifestDescriptor: md, + } + res = append(res, reference) + } + return res, nil +} + +// Close deletes temporary files associated with the Reader, if any. +func (r *Reader) Close() error { + return os.RemoveAll(r.tempDirectory) +} diff --git a/oci/archive/writer.go b/oci/archive/writer.go new file mode 100644 index 0000000000..989474fba8 --- /dev/null +++ b/oci/archive/writer.go @@ -0,0 +1,60 @@ +package archive + +import ( + "context" + "fmt" + "io/ioutil" + "os" + + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/oci/layout" + "github.com/containers/image/v5/types" +) + +// Writer keeps the tempDir for creating oci archive and archive destination +type Writer struct { + // tempDir will be tarred to oci archive + tempDir string + // user-specified path + path string +} + +// NewWriter creates a temp directory will be tarred to oci-archive. +// The caller should call .Close() on the returned object. +func NewWriter(ctx context.Context, sys *types.SystemContext, file string) (*Writer, error) { + dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci") + if err != nil { + return nil, fmt.Errorf("%w: error creating temp directory", err) + } + ociWriter := &Writer{ + tempDir: dir, + path: file, + } + return ociWriter, nil +} + +// NewReference returns an ImageReference that allows adding an image to Writer, +// with an optional image name +func (w *Writer) NewReference(name string) (types.ImageReference, error) { + ref, err := layout.NewReference(w.tempDir, name) + if err != nil { + return nil, fmt.Errorf("%w: error creating image reference", err) + } + if ref == nil { + ref, err = newReference(w.path, name, -1, nil, w) + if err != nil { + return nil, fmt.Errorf("%w: error creating image reference", err) + } + } + return ref, nil +} + +// Close converts the data about images in the temp directory to the archive and +// deletes temporary files associated with the Writer +func (w *Writer) Close() error { + err := tarDirectory(w.tempDir, w.path) + if err2 := os.RemoveAll(w.tempDir); err2 != nil && err == nil { + err = err2 + } + return err +} diff --git a/oci/internal/oci_util.go b/oci/internal/oci_util.go index c2012e50e0..af873a0056 100644 --- a/oci/internal/oci_util.go +++ b/oci/internal/oci_util.go @@ -1,11 +1,14 @@ package internal import ( - "github.com/pkg/errors" "path/filepath" "regexp" "runtime" + "strconv" "strings" + + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) // annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys @@ -124,3 +127,45 @@ func validateScopeNonWindows(scope string) error { return nil } + +// ParseOCIReferenceName parses the image from the oci reference that contains an index. +func ParseOCIReferenceName(image string) (img string, index int, err error) { + index = -1 + if strings.HasPrefix(image, "@") { + idx, err := strconv.Atoi(image[1:]) + if err != nil { + return "", index, errors.Wrapf(err, "Invalid source index @%s: not an integer", image[1:]) + } + if idx < 0 { + return "", index, errors.Errorf("Invalid source index @%d: must not be negative", idx) + } + index = idx + } else { + img = image + } + return img, index, nil +} + +// ParseReferenceIntoElements splits the oci reference into location, image name and source index if exists +func ParseReferenceIntoElements(reference string) (string, string, int, error) { + dir, image := SplitPathAndImage(reference) + image, index, err := ParseOCIReferenceName(image) + if err != nil { + return "", "", -1, err + } + return dir, image, index, nil +} + +// NameFromAnnotations returns a reference string to be used as an image name, +// or an empty string. The annotations map may be nil. +func NameFromAnnotations(annotations map[string]string) string { + if annotations == nil { + return "" + } + // buildkit/containerd are using a custom annotation see + // containers/podman/issues/12560. + if annotations["io.containerd.image.name"] != "" { + return annotations["io.containerd.image.name"] + } + return annotations[imgspecv1.AnnotationRefName] +} diff --git a/oci/internal/oci_util_test.go b/oci/internal/oci_util_test.go index b10071d2d7..59d66fbc2e 100644 --- a/oci/internal/oci_util_test.go +++ b/oci/internal/oci_util_test.go @@ -2,8 +2,9 @@ package internal import ( "fmt" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) type testDataSplitReference struct { @@ -17,6 +18,12 @@ type testDataScopeValidation struct { errMessage string } +type testOCIReference struct { + ref string + image string + index int +} + func TestSplitReferenceIntoDirAndImageWindows(t *testing.T) { tests := []testDataSplitReference{ {`C:\foo\bar:busybox:latest`, `C:\foo\bar`, "busybox:latest"}, @@ -60,3 +67,25 @@ func TestValidateScopeWindows(t *testing.T) { } } } + +func TestParseOCIReferenceName(t *testing.T) { + validTests := []testOCIReference{ + {"@0", "", 0}, + {"notlatest@1", "notlatest@1", -1}, + } + for _, test := range validTests { + img, idx, err := ParseOCIReferenceName(test.ref) + assert.NoError(t, err) + assert.Equal(t, img, test.image) + assert.Equal(t, idx, test.index) + } + + invalidTests := []string{ + "@-5", + "@invalidIndex", + } + for _, test := range invalidTests { + _, _, err := ParseOCIReferenceName(test) + assert.Error(t, err) + } +} diff --git a/oci/layout/fixtures/two_names_manifest/index.json b/oci/layout/fixtures/two_names_manifest/index.json new file mode 100644 index 0000000000..8947602ddb --- /dev/null +++ b/oci/layout/fixtures/two_names_manifest/index.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + }, + "annotations": { + "org.opencontainers.image.ref.name": "imageValue0" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:62ae1939cdb49e93f32cdb29d61ad276552532c6ae7fd543de7bf22511e4965e", + "size": 348, + "annotations": { + "org.opencontainers.image.ref.name": "imageValue1" + } + } + ] +} diff --git a/oci/layout/oci_dest.go b/oci/layout/oci_dest.go index d0ee726352..379bad560f 100644 --- a/oci/layout/oci_dest.go +++ b/oci/layout/oci_dest.go @@ -27,6 +27,9 @@ type ociImageDestination struct { // newImageDestination returns an ImageDestination for writing to an existing directory. func newImageDestination(sys *types.SystemContext, ref ociReference) (types.ImageDestination, error) { + if ref.sourceIndex != -1 { + return nil, errors.Errorf("Destination reference must not contain a manifest index @%d", ref.sourceIndex) + } var index *imgspecv1.Index if indexExists(ref) { var err error diff --git a/oci/layout/oci_transport.go b/oci/layout/oci_transport.go index a99b631584..2398218e77 100644 --- a/oci/layout/oci_transport.go +++ b/oci/layout/oci_transport.go @@ -61,22 +61,31 @@ type ociReference struct { // (But in general, we make no attempt to be completely safe against concurrent hostile filesystem modifications.) dir string // As specified by the user. May be relative, contain symlinks, etc. resolvedDir string // Absolute path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces. - // If image=="", it means the "only image" in the index.json is used in the case it is a source - // for destinations, the image name annotation "image.ref.name" is not added to the index.json + // If image=="" && sourceIndex==-1, it means the "only image" in the index.json is used in the case it is a source + // for destinations, the image name annotation "image.ref.name" is not added to the index.json. + // + // Must not be set if sourceIndex is set (the value is not -1). image string + // If not -1, a zero-based index of an image in the manifest index. Valid only for sources. + // Must not be set if image is set. + sourceIndex int } // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { - dir, image := internal.SplitPathAndImage(reference) - return NewReference(dir, image) + dir, image, index, err := internal.ParseReferenceIntoElements(reference) + if err != nil { + return nil, err + } + return newReference(dir, image, index) } -// NewReference returns an OCI reference for a directory and a image. +// newReference returns an OCI reference for a directory and a imageļ¼Œ sourceIndex points to an image. // +// If sourceIndex==-1, the index will not be valid to point out the source image, only image will be used. // We do not expose an API supplying the resolvedDir; we could, but recomputing it // is generally cheap enough that we prefer being confident about the properties of resolvedDir. -func NewReference(dir, image string) (types.ImageReference, error) { +func newReference(dir, image string, sourceIndex int) (types.ImageReference, error) { resolved, err := explicitfilepath.ResolvePathToFullyExplicit(dir) if err != nil { return nil, err @@ -90,7 +99,26 @@ func NewReference(dir, image string) (types.ImageReference, error) { return nil, err } - return ociReference{dir: dir, resolvedDir: resolved, image: image}, nil + if sourceIndex != -1 && sourceIndex < 0 { + return nil, fmt.Errorf("Invalid oci layout: reference: index @%d must not be negative", sourceIndex) + } + if sourceIndex != -1 && image != "" { + return nil, fmt.Errorf("Invalid oci layout: reference: cannot use both an image %s and a source index @%d", image, sourceIndex) + } + return ociReference{dir: dir, resolvedDir: resolved, image: image, sourceIndex: sourceIndex}, nil +} + +// NewIndexReference returns an OCI reference for a path and a zero-based source manifest index. +func NewIndexReference(dir string, sourceIndex int) (types.ImageReference, error) { + return newReference(dir, "", sourceIndex) +} + +// NewReference returns an OCI reference for a directory and a image. +// +// We do not expose an API supplying the resolvedDir; we could, but recomputing it +// is generally cheap enough that we prefer being confident about the properties of resolvedDir. +func NewReference(dir, image string) (types.ImageReference, error) { + return newReference(dir, image, -1) } func (ref ociReference) Transport() types.ImageTransport { @@ -103,7 +131,11 @@ func (ref ociReference) Transport() types.ImageTransport { // e.g. default attribute values omitted by the user may be filled in in the return value, or vice versa. // WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix. func (ref ociReference) StringWithinTransport() string { - return fmt.Sprintf("%s:%s", ref.dir, ref.image) + if ref.sourceIndex == -1 { + return fmt.Sprintf("%s:%s", ref.dir, ref.image) + } + return fmt.Sprintf("%s:@%d", ref.dir, ref.sourceIndex) + } // DockerReference returns a Docker reference associated with this reference @@ -182,8 +214,16 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) { if err != nil { return imgspecv1.Descriptor{}, err } - var d *imgspecv1.Descriptor + + if ref.sourceIndex != -1 { + if ref.sourceIndex < len(index.Manifests) { + d = &index.Manifests[ref.sourceIndex] + return *d, nil + } else { + return imgspecv1.Descriptor{}, fmt.Errorf("index %d is too large, only %d entries available", ref.sourceIndex, len(index.Manifests)) + } + } if ref.image == "" { // return manifest if only one image is in the oci directory if len(index.Manifests) == 1 { @@ -198,8 +238,8 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) { if md.MediaType != imgspecv1.MediaTypeImageManifest && md.MediaType != imgspecv1.MediaTypeImageIndex { continue } - refName, ok := md.Annotations[imgspecv1.AnnotationRefName] - if !ok { + refName := internal.NameFromAnnotations(md.Annotations) + if refName == "" { continue } if refName == ref.image { @@ -219,7 +259,7 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) { func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor, error) { ociRef, ok := imgRef.(ociReference) if !ok { - return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociRef") + return imgspecv1.Descriptor{}, fmt.Errorf("error typecasting, need type ociRef") } return ociRef.getManifestDescriptor() } @@ -238,7 +278,7 @@ func (ref ociReference) NewImageDestination(ctx context.Context, sys *types.Syst // DeleteImage deletes the named image from the registry, if supported. func (ref ociReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { - return errors.Errorf("Deleting images not implemented for oci: images") + return fmt.Errorf("Deleting images not implemented for oci: images") } // ociLayoutPath returns a path for the oci-layout within a directory using OCI conventions. @@ -254,7 +294,7 @@ func (ref ociReference) indexPath() string { // blobPath returns a path for a blob within a directory using OCI image-layout conventions. func (ref ociReference) blobPath(digest digest.Digest, sharedBlobDir string) (string, error) { if err := digest.Validate(); err != nil { - return "", errors.Wrapf(err, "unexpected digest reference %s", digest) + return "", fmt.Errorf("%w: unexpected digest reference %s", err, digest) } blobDir := filepath.Join(ref.dir, "blobs") if sharedBlobDir != "" { diff --git a/oci/layout/oci_transport_test.go b/oci/layout/oci_transport_test.go index a795e9dd3a..2b64edd9d7 100644 --- a/oci/layout/oci_transport_test.go +++ b/oci/layout/oci_transport_test.go @@ -13,16 +13,28 @@ import ( "github.com/stretchr/testify/require" ) -// TestGetManifestDescriptor is testing a regression issue where a nil error was being wrapped, -// this causes the returned error to be nil as well and the user wasn't getting a proper error output. -// -// More info: https://github.com/containers/skopeo/issues/496 func TestGetManifestDescriptor(t *testing.T) { imageRef, err := NewReference("fixtures/two_images_manifest", "") require.NoError(t, err) + // test a regression issue where a nil error was being wrapped, + // this causes the returned error to be nil as well and the user wasn't getting a proper error output. + // + // More info: https://github.com/containers/skopeo/issues/496 _, err = imageRef.(ociReference).getManifestDescriptor() assert.EqualError(t, err, ErrMoreThanOneImage.Error()) + + imageRef, err = NewReference("fixtures/two_names_manifest", "imageValue0") + require.NoError(t, err) + manDescriptor, err := imageRef.(ociReference).getManifestDescriptor() + require.NoError(t, err) + assert.Equal(t, manDescriptor.Annotations["org.opencontainers.image.ref.name"], "imageValue0") + + imageRef, err = NewIndexReference("fixtures/two_names_manifest", 1) + require.NoError(t, err) + manDescriptor, err = imageRef.(ociReference).getManifestDescriptor() + require.NoError(t, err) + assert.Equal(t, manDescriptor.Annotations["org.opencontainers.image.ref.name"], "imageValue1") } func TestTransportName(t *testing.T) { @@ -124,6 +136,10 @@ func TestNewReference(t *testing.T) { _, err = NewReference(tmpDir+"/has:colon", imageValue) assert.Error(t, err) + + // Test private newReference + _, err = newReference(tmpDir, imageValue, 1) + assert.Error(t, err) } // refToTempOCI creates a temporary directory and returns an reference to it. @@ -170,7 +186,8 @@ func TestReferenceStringWithinTransport(t *testing.T) { for _, c := range []struct{ input, result string }{ {"/dir1:notlatest:notlatest", "/dir1:notlatest:notlatest"}, // Explicit image - {"/dir3:", "/dir3:"}, // No image + {"/dir3:", "/dir3:"}, // No image + {"/dir1:@1", "/dir1:@1"}, // Explicit sourceIndex of image } { ref, err := ParseReference(tmpDir + c.input) require.NoError(t, err, c.input)