From 139b1592d03441adfc1ac72b4eece2324af38702 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Mon, 22 Jan 2024 11:35:00 -0500 Subject: [PATCH] libimage/manifests.list: hold artifact manifests Add an AddArtifact() method which will craft an artifact manifest which references one or more files and then add that manifest to the index. When we need to build a reference to a list that includes our artifact manifests, save the manifests and symlinks to their "layer" blobs along with the contents of inlined blobs in a per-image directory, and add references to those locations to the list of images we can search for manifests and blobs. Signed-off-by: Nalin Dahyabhai --- libimage/manifests/manifests.go | 375 ++++++++++++++++++++++++++- libimage/manifests/manifests_test.go | 343 +++++++++++++++++++++++- 2 files changed, 713 insertions(+), 5 deletions(-) diff --git a/libimage/manifests/manifests.go b/libimage/manifests/manifests.go index 130abd3a1..439f77adb 100644 --- a/libimage/manifests/manifests.go +++ b/libimage/manifests/manifests.go @@ -1,15 +1,20 @@ package manifests import ( + "bytes" "context" "encoding/json" "errors" "fmt" "io" + "mime" + "net/http" "os" + "path/filepath" "strings" "time" + "github.com/containers/common/internal" "github.com/containers/common/pkg/manifests" "github.com/containers/common/pkg/retry" "github.com/containers/common/pkg/supplemented" @@ -17,6 +22,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/image" "github.com/containers/image/v5/manifest" + ocilayout "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/pkg/compression" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/signature/signer" @@ -25,8 +31,10 @@ import ( "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" "github.com/containers/storage" + "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/lockfile" digest "github.com/opencontainers/go-digest" + imgspec "github.com/opencontainers/image-spec/specs-go" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" "golang.org/x/exp/slices" @@ -37,8 +45,9 @@ const ( ) const ( - instancesData = "instances.json" - artifactsData = "artifacts.json" + instancesData = "instances.json" + artifactsData = "artifacts.json" + pushingArtifactsSubdirectory = "referenced-artifacts" ) // LookupReferenceFunc return an image reference based on the specified one. @@ -74,6 +83,7 @@ type List interface { Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error) Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error) + AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error) } // PushOptions includes various settings which are needed for pushing the @@ -222,7 +232,7 @@ func (l *list) SaveToImage(store storage.Store, imageID string, names []string, // in the list, or an error if the list has never been saved to a local image. func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) { if l.instances[""] == "" { - return nil, fmt.Errorf("building reference to list: %w", ErrListImageUnknown) + return nil, fmt.Errorf("building reference to list, appears to have not been saved first: %w", ErrListImageUnknown) } s, err := is.Transport.ParseStoreReference(store, l.instances[""]) if err != nil { @@ -246,6 +256,92 @@ func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, in } } } + if len(l.artifacts.Manifests) > 0 { + img, err := is.Transport.GetImage(s) + if err != nil { + return nil, fmt.Errorf("locating image %s: %w", transports.ImageName(s), err) + } + imgDirectory, err := store.ImageDirectory(img.ID) + if err != nil { + return nil, fmt.Errorf("locating per-image directory for %s: %w", img.ID, err) + } + tmp, err := os.MkdirTemp(imgDirectory, pushingArtifactsSubdirectory) + if err != nil { + return nil, err + } + for artifactManifestDigest, contents := range l.artifacts.Manifests { + // create the blobs directory + blobsDir := filepath.Join(tmp, "blobs", artifactManifestDigest.Algorithm().String()) + if err := os.MkdirAll(blobsDir, 0o700); err != nil { + return nil, fmt.Errorf("creating directory for blobs: %w", err) + } + // write the artifact manifest + if err := os.WriteFile(filepath.Join(blobsDir, artifactManifestDigest.Encoded()), []byte(contents), 0o644); err != nil { + return nil, fmt.Errorf("writing artifact manifest as blob: %w", err) + } + // symlink all of the referenced files and write the inlined blobs into the blobs directory + var referencedBlobDigests []digest.Digest + var symlinkedFiles []string + if referencedConfigDigest, ok := l.artifacts.Configs[artifactManifestDigest]; ok { + referencedBlobDigests = append(referencedBlobDigests, referencedConfigDigest) + } + referencedBlobDigests = append(referencedBlobDigests, l.artifacts.Layers[artifactManifestDigest]...) + for _, referencedBlobDigest := range referencedBlobDigests { + referencedFile, knownFile := l.artifacts.Detached[referencedBlobDigest] + referencedBlob, knownBlob := l.artifacts.Blobs[referencedBlobDigest] + if !knownFile && !knownBlob { + return nil, fmt.Errorf(`internal error: no file or blob with artifact "layer" digest %q recorded`, referencedBlobDigest) + } + expectedLayerBlobPath := filepath.Join(blobsDir, referencedBlobDigest.Encoded()) + if knownFile { + if err := os.Symlink(referencedFile, expectedLayerBlobPath); err != nil { + return nil, err + } + symlinkedFiles = append(symlinkedFiles, referencedFile) + } + if knownBlob { + if err := os.WriteFile(expectedLayerBlobPath, referencedBlob, 0o600); err != nil { + return nil, err + } + } + } + // write the index that refers to this one artifact image + tag := "latest" + indexFile := filepath.Join(tmp, "index.json") + index := v1.Index{ + Versioned: imgspec.Versioned{ + SchemaVersion: 2, + }, + MediaType: v1.MediaTypeImageIndex, + Manifests: []v1.Descriptor{{ + MediaType: v1.MediaTypeImageManifest, + Digest: artifactManifestDigest, + Size: int64(len(contents)), + Annotations: map[string]string{ + v1.AnnotationRefName: tag, + }, + }}, + } + indexBytes, err := json.Marshal(&index) + if err != nil { + return nil, fmt.Errorf("encoding image index for OCI layout: %w", err) + } + if err := os.WriteFile(indexFile, indexBytes, 0o644); err != nil { + return nil, fmt.Errorf("writing image index for OCI layout: %w", err) + } + // write the layout file + layoutFile := filepath.Join(tmp, "oci-layout") + if err := os.WriteFile(layoutFile, []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644); err != nil { + return nil, fmt.Errorf("writing oci-layout file: %w", err) + } + // build the reference to this artifact image's oci layout + ref, err := ocilayout.NewReference(tmp, tag) + if err != nil { + return nil, fmt.Errorf("creating ImageReference for artifact with files %q: %w", symlinkedFiles, err) + } + references = append(references, ref) + } + } for _, instance := range whichInstances { imageName := l.instances[instance] ref, err := alltransports.ParseImageName(imageName) @@ -522,6 +618,279 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag return manifestDigest, nil } +// AddArtifactOptions contains options which control the contents of the +// artifact manifest that AddArtifact will create and add to the image index. + +// This should provide for all of the ways to construct a manifest outlined in +// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage +// * no blobs ￫ set ManifestArtifactType +// * blobs, no configuration ￫ set ManifestArtifactType and possibly LayerMediaType, and provide file names +// * blobs and configuration ￫ set ManifestArtifactType, possibly LayerMediaType, and ConfigDescriptor, and provide file names +// +// The older style of describing artifacts: +// * leave ManifestArtifactType blank +// * specify a zero-length application/vnd.oci.image.config.v1+json config blob +// * set LayerMediaType to a custom type +// +// When reading data produced elsewhere, note that newer tooling will produce +// manifests with ArtifactType set. If the manifest's ArtifactType is not set, +// consumers should consult the config descriptor's MediaType. +type AddArtifactOptions struct { + ManifestArtifactType *string // overall type of the artifact manifest. default: "application/vnd.unknown.artifact.v1" + ManifestMediaType *string // default: the OCI image media type + Platform v1.Platform // default: add to the index without platform information + ConfigDescriptor *v1.Descriptor // default: a descriptor for an explicitly empty config blob + ConfigFile string // path to config contents, recorded if ConfigDescriptor.Size != 0 and ConfigDescriptor.Data is not set + LayerMediaType *string // default: mime.TypeByExtension() if basename contains ".", else http.DetectContentType() + LayerArtifactType *string // optional, default is "" + Annotations map[string]string // optional, default is none + SubjectReference types.ImageReference // optional + ExcludeTitles bool // don't add "org.opencontainers.image.title" annotations set to file base names +} + +// AddArtifact creates an artifact manifest describing the specified file or +// files, then adds them to the specified image index. Returns the +// instanceDigest for the artifact manifest. +// The caller could craft the manifest themselves and use Add() to add it to +// the image index and get the same end-result, but this should save them some +// work. +func (l *list) AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error) { + // If we were given a subject, build a descriptor for it first, since + // it might be remote, and anything else we do before looking at it + // might have to get thrown away if we can't get to it for whatever + // reason. + var subject *v1.Descriptor + if options.SubjectReference != nil { + subjectReference, err := options.SubjectReference.NewImageSource(ctx, sys) + if err != nil { + return "", fmt.Errorf("setting up to read manifest and configuration from subject %q: %w", transports.ImageName(options.SubjectReference), err) + } + defer subjectReference.Close() + subjectManifestBytes, subjectManifestType, err := subjectReference.GetManifest(ctx, nil) + if err != nil { + return "", fmt.Errorf("reading manifest from subject %q: %w", transports.ImageName(options.SubjectReference), err) + } + subjectManifestDigest, err := manifest.Digest(subjectManifestBytes) + if err != nil { + return "", fmt.Errorf("digesting manifest of subject %q: %w", transports.ImageName(options.SubjectReference), err) + } + subject = &v1.Descriptor{ + MediaType: subjectManifestType, + Digest: subjectManifestDigest, + Size: int64(len(subjectManifestBytes)), + } + } + + // Build up the layers list piece by piece. + var layers []v1.Descriptor + fileDigests := make(map[string]digest.Digest) + + if len(files) == 0 { + // https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage + // says that we should have at least one layer listed, even if it's just a placeholder + layers = append(layers, v1.DescriptorEmptyJSON) + } + for _, file := range files { + if err := func() error { + // Open the file so that we can digest it. + absFile, err := filepath.Abs(file) + if err != nil { + return fmt.Errorf("converting %q to an absolute path: %w", file, err) + } + + f, err := os.Open(absFile) + if err != nil { + return fmt.Errorf("reading %q to determine its digest: %w", file, err) + } + defer f.Close() + + // Hang on to a copy of the first 512 bytes, but digest the whole thing. + digester := digest.Canonical.Digester() + writeCounter := ioutils.NewWriteCounter(digester.Hash()) + var detectableData bytes.Buffer + _, err = io.CopyN(writeCounter, io.TeeReader(f, &detectableData), 512) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("reading %q to determine its digest: %w", file, err) + } + if err == nil { + if _, err := io.Copy(writeCounter, f); err != nil { + return fmt.Errorf("reading %q to determine its digest: %w", file, err) + } + } + fileDigests[absFile] = digester.Digest() + + // If one wasn't specified, figure out what the MediaType should be. + title := filepath.Base(absFile) + layerMediaType := options.LayerMediaType + if layerMediaType == nil { + if index := strings.LastIndex(title, "."); index != -1 { + // File's basename has an extension, try to use a shortcut. + tmp := mime.TypeByExtension(title[index:]) + if tmp != "" { + layerMediaType = &tmp + } + } + if layerMediaType == nil { + // File's basename has no extension or didn't map to a type, look at the contents we saved. + tmp := http.DetectContentType(detectableData.Bytes()) + layerMediaType = &tmp + } + if layerMediaType != nil { + // Strip off any parameters, since we only want the type name. + if parsedMediaType, _, err := mime.ParseMediaType(*layerMediaType); err == nil { + layerMediaType = &parsedMediaType + } + } + } + + // Only set an ArtifactType for the layer if one was specified. + layerArtifactType := options.LayerArtifactType + if layerArtifactType == nil { + tmp := "" + layerArtifactType = &tmp + } + descriptor := v1.Descriptor{ + MediaType: *layerMediaType, + ArtifactType: *layerArtifactType, + Digest: fileDigests[absFile], + Size: writeCounter.Count, + } + // Add a title annotation unless we were explicitly told not to. + if !options.ExcludeTitles { + descriptor.Annotations = map[string]string{ + v1.AnnotationTitle: title, + } + } + layers = append(layers, descriptor) + return nil + }(); err != nil { + return "", err + } + } + + // Unless we were explicitly told otherwise, default to the normal image MediaType. + mediaType := v1.MediaTypeImageManifest + if options.ManifestMediaType != nil && *options.ManifestMediaType != "" { + mediaType = *options.ManifestMediaType + } + + // Unless we were told what this is, use the default that ORAS uses. + artifactType := "application/vnd.unknown.artifact.v1" + if options.ManifestArtifactType != nil { + artifactType = *options.ManifestArtifactType + } + + // Unless we were explicitly told otherwise, default to an empty config blob. + configDescriptor := internal.DeepCopyDescriptor(&v1.DescriptorEmptyJSON) + if options.ConfigDescriptor != nil { + configDescriptor = internal.DeepCopyDescriptor(options.ConfigDescriptor) + } else if options.ConfigFile != "" { + configDescriptor = &v1.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Digest: "", // to be figured out below + Size: -1, // to be figured out below + } + } + configFilePath := "" + if configDescriptor.Size != 0 { + if len(configDescriptor.Data) == 0 { + if options.ConfigFile == "" { + return "", fmt.Errorf("needed config data file, but none was provided") + } + filePath, err := filepath.Abs(options.ConfigFile) + if err != nil { + return "", fmt.Errorf("recording artifact config data file %q: %w", options.ConfigFile, err) + } + digester := digest.Canonical.Digester() + counter := ioutils.NewWriteCounter(digester.Hash()) + if err := func() error { + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("reading artifact config data file %q: %w", options.ConfigFile, err) + } + defer f.Close() + if _, err := io.Copy(counter, f); err != nil { + return fmt.Errorf("digesting artifact config data file %q: %w", options.ConfigFile, err) + } + return nil + }(); err != nil { + return "", err + } + configDescriptor.Data = nil + configDescriptor.Size = counter.Count + configDescriptor.Digest = digester.Digest() + configFilePath = filePath + } else { + decoder := bytes.NewReader(configDescriptor.Data) + digester := digest.Canonical.Digester() + counter := ioutils.NewWriteCounter(digester.Hash()) + if _, err := io.Copy(counter, decoder); err != nil { + return "", fmt.Errorf("digesting inlined artifact config data: %w", err) + } + configDescriptor.Size = counter.Count + configDescriptor.Digest = digester.Digest() + } + } else { + configDescriptor.Digest = digest.Canonical.FromString("") + } + + // Construct the manifest. + artifactManifest := v1.Manifest{ + Versioned: imgspec.Versioned{ + SchemaVersion: 2, + }, + MediaType: mediaType, + ArtifactType: artifactType, + Config: *configDescriptor, + Layers: layers, + Subject: subject, + } + // Add in annotations, more or less exactly as specified. + if options.Annotations != nil { + artifactManifest.Annotations = make(map[string]string) + } + for k, v := range options.Annotations { + artifactManifest.Annotations[k] = v + } + + // Encode and save the data we care about. + artifactManifestBytes, err := json.Marshal(artifactManifest) + if err != nil { + return "", fmt.Errorf("marshalling the artifact manifest: %w", err) + } + artifactManifestDigest, err := manifest.Digest(artifactManifestBytes) + if err != nil { + return "", fmt.Errorf("digesting the artifact manifest: %w", err) + } + l.artifacts.Manifests[artifactManifestDigest] = string(artifactManifestBytes) + l.artifacts.Layers[artifactManifestDigest] = nil + if configFilePath != "" { + l.artifacts.Configs[artifactManifestDigest] = artifactManifest.Config.Digest + l.artifacts.Detached[artifactManifest.Config.Digest] = configFilePath + l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], configFilePath) + } + if len(artifactManifest.Config.Data) != 0 { + l.artifacts.Configs[artifactManifestDigest] = artifactManifest.Config.Digest + l.artifacts.Blobs[artifactManifest.Config.Digest] = slices.Clone(artifactManifest.Config.Data) + } + for filePath, fileDigest := range fileDigests { + l.artifacts.Layers[artifactManifestDigest] = append(l.artifacts.Layers[artifactManifestDigest], fileDigest) + l.artifacts.Detached[fileDigest] = filePath + l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], filePath) + } + // Add this artifact manifest to the image index. + if err := l.AddInstance(artifactManifestDigest, int64(len(artifactManifestBytes)), artifactManifest.MediaType, options.Platform.OS, options.Platform.Architecture, options.Platform.OSVersion, options.Platform.OSFeatures, options.Platform.Variant, nil, nil); err != nil { + return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err) + } + // Set the artifact type in the image index entry if we have one, since AddInstance() didn't do that for us. + if artifactManifest.ArtifactType != "" { + if err := l.List.SetArtifactType(&artifactManifestDigest, artifactManifest.ArtifactType); err != nil { + return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err) + } + } + return artifactManifestDigest, nil +} + // Remove filters out any instances in the list which match the specified digest. func (l *list) Remove(instanceDigest digest.Digest) error { err := l.List.Remove(instanceDigest) diff --git a/libimage/manifests/manifests_test.go b/libimage/manifests/manifests_test.go index ab20cb511..2f71ec8b2 100644 --- a/libimage/manifests/manifests_test.go +++ b/libimage/manifests/manifests_test.go @@ -3,22 +3,32 @@ package manifests import ( "bytes" "context" + "encoding/json" "fmt" + "io" + "os" "path/filepath" + "runtime" + "sort" + "strings" "testing" "time" + "github.com/containerd/containerd/platforms" "github.com/containers/common/pkg/manifests" cp "github.com/containers/image/v5/copy" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/compression" + "github.com/containers/image/v5/transports" "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" "github.com/containers/storage" "github.com/containers/storage/pkg/unshare" digest "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var ( @@ -163,6 +173,318 @@ func TestAddRemove(t *testing.T) { assert.Equalf(t, len(list.Instances()), 1, "too many instances added", otherListInstanceDigest) } +func TestAddArtifact(t *testing.T) { + if unshare.IsRootless() { + t.Skip("Test can only run as root") + } + ctx := context.Background() + dir := t.TempDir() + storeOptions := storage.StoreOptions{ + GraphRoot: filepath.Join(dir, "root"), + RunRoot: filepath.Join(dir, "runroot"), + GraphDriverName: "vfs", + } + emptyConfigFile := filepath.Join(dir, "empty.json") + err := os.WriteFile(emptyConfigFile, []byte("{}"), 0o600) + assert.NoError(t, err, "error creating a mostly-empty file") + store, err := storage.GetStore(storeOptions) + assert.NoError(t, err, "error opening store") + if store == nil { + return + } + defer func() { + if _, err := store.Shutdown(true); err != nil { + assert.NoError(t, err, "error closing store") + } + }() + subjectImageName := "oci-archive:" + filepath.Join("..", "testdata", "oci-name-only.tar.gz") + subjectReference, err := alltransports.ParseImageName(subjectImageName) + require.NoError(t, err) + testCombination := func(t *testing.T, file, fileMediaType string, manifestMediaType, manifestArtifactType *string, platform v1.Platform, configDescriptor *v1.Descriptor, configFile string, layerMediaType, layerArtifactType *string, annotations map[string]string, subjectReference types.ImageReference, excludeTitles bool) { + subjectName := "" + if subjectReference != nil { + subjectName = transports.ImageName(subjectReference) + } + configMediaType := "" + if configDescriptor != nil { + configMediaType = configDescriptor.MediaType + } + var platformStr string + if platform.OS != "" && platform.Architecture != "" { + platformStr = platforms.Format(platform) + } + manifestMediaTypeStr, manifestArtifactTypeStr := "", "" + if manifestMediaType != nil { + manifestMediaTypeStr = *manifestMediaType + } + if manifestArtifactType != nil { + manifestArtifactTypeStr = *manifestArtifactType + } + layerMediaTypeStr, layerArtifactTypeStr := "", "" + if layerMediaType != nil { + layerMediaTypeStr = *layerMediaType + } + if layerArtifactType != nil { + layerArtifactTypeStr = *layerArtifactType + } + annotationsStr := "" + if annotations != nil { + var annotationsSlice []string + for k, v := range annotations { + annotationsSlice = append(annotationsSlice, fmt.Sprintf("%q=%q", k, v)) + } + sort.Strings(annotationsSlice) + annotationsStr = "{" + strings.Join(annotationsSlice, ",") + "}" + } + fileBasename := "" + if file != "" { + fileBasename = filepath.Base(file) + } + desc := fmt.Sprint("file=", fileBasename, ",fileMediaType=", fileMediaType, ",manifestMediaType=", manifestMediaTypeStr, ",manifestArtifactType=", manifestArtifactTypeStr, ",platform=", platformStr, ",configMediaType=", configMediaType, ",configFile=", configFile, ",layerMediaType=", layerMediaTypeStr, ",layerArtifactType=", layerArtifactTypeStr, ",annotations=", annotationsStr, ",subject=", subjectName, ",excludeTitles=", excludeTitles) + t.Run(desc, func(t *testing.T) { + // create the new index and add the file to it + options := AddArtifactOptions{ + ManifestArtifactType: manifestArtifactType, + ManifestMediaType: manifestMediaType, + Platform: platform, + ConfigDescriptor: configDescriptor, + ConfigFile: configFile, + LayerMediaType: layerMediaType, + LayerArtifactType: layerArtifactType, + Annotations: annotations, + SubjectReference: subjectReference, + ExcludeTitles: excludeTitles, + } + list := Create() + var instanceDigest digest.Digest + if file != "" { + instanceDigest, err = list.AddArtifact(ctx, sys, options, file) + } else { + instanceDigest, err = list.AddArtifact(ctx, sys, options) + } + assert.NoErrorf(t, err, "list.AddArtifact(%#v)", options) + assert.Equal(t, 1, len(list.Instances()), "too many instances added") + // have to save it before we can create a reference to it + _, err = list.SaveToImage(store, "", nil, "") + require.NoError(t, err) + // get ready to copy it, sort of + ref, err := list.Reference(store, cp.CopyAllImages, nil) + require.NoError(t, err) + // fetch the manifest for the artifact that we just added to the index + src, err := ref.NewImageSource(ctx, &types.SystemContext{}) + require.NoError(t, err) + defer src.Close() + manifestBytes, manifestType, err := src.GetManifest(ctx, &instanceDigest) + require.NoError(t, err) + // decode the artifact manifest + var m v1.Manifest + if annotations != nil { + m.Annotations = make(map[string]string) + } + err = json.Unmarshal(manifestBytes, &m) + require.NoError(t, err) + // check that the artifact manifest looks right + expectedManifestMediaType := v1.MediaTypeImageManifest + if manifestMediaType != nil && *manifestMediaType != "" { + expectedManifestMediaType = *manifestMediaType + } + assert.Equal(t, expectedManifestMediaType, manifestType) + assert.Equal(t, expectedManifestMediaType, m.MediaType) + expectedManifestArtifactType := "application/vnd.unknown.artifact.v1" + if manifestArtifactType != nil { + expectedManifestArtifactType = *manifestArtifactType + } + assert.Equal(t, expectedManifestArtifactType, m.ArtifactType) + // check the config blob info + configFileSize := v1.DescriptorEmptyJSON.Size + configFileDigest := v1.DescriptorEmptyJSON.Digest + if configFile != "" { + f, err := os.Open(configFile) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, f.Close()) }) + st, err := f.Stat() + require.NoError(t, err) + configFileSize = st.Size() + digester := digest.Canonical.Digester() + _, err = io.Copy(digester.Hash(), f) + require.NoError(t, err) + configFileDigest = digester.Digest() + } + if configDescriptor != nil && configFile != "" { + assert.Equal(t, configDescriptor.MediaType, m.Config.MediaType, "did not record expected media type for config with file") + assert.Equal(t, configFileDigest, m.Config.Digest, "did not record expected digest for config with file") + assert.Equal(t, configFileSize, m.Config.Size, "did not record expected size for config with file") + } else if configFile != "" { + assert.Equal(t, v1.MediaTypeImageConfig, m.Config.MediaType, "did not record expected media type for config file") + assert.Equal(t, configFileDigest, m.Config.Digest, "did not record expected digest for config with file") + assert.Equal(t, configFileSize, m.Config.Size, "did not record expected size for config with file") + } else if configDescriptor != nil { + assert.Equal(t, configDescriptor.MediaType, m.Config.MediaType, "did not record expected mediaType for empty config") + if false { + d := configDescriptor.Digest + s := configDescriptor.Size + if d.Validate() != nil { + s = 0 + d = digest.Canonical.FromString("") + } + assert.Equal(t, d, m.Config.Digest, "did not record expected digest for empty config") + assert.Equal(t, s, m.Config.Size, "did not record expected digest for empty config") + } + assert.Equal(t, configDescriptor.Digest, m.Config.Digest, "did not record expected digest for empty config") + assert.Equal(t, configDescriptor.Size, m.Config.Size, "did not record expected digest for empty config") + } else { + assert.Equal(t, v1.DescriptorEmptyJSON.MediaType, m.Config.MediaType, "did not record expected mediaType for empty config and no config file") + assert.Equal(t, v1.DescriptorEmptyJSON.Digest, m.Config.Digest, "did not record expected digest for empty config and no config file") + assert.Equal(t, v1.DescriptorEmptyJSON.Size, m.Config.Size, "did not record expected digest for empty config and no config file") + } + // if we had a file, it should be there as the "layer", otherwise it should be the empty descriptor + assert.Equal(t, 1, len(m.Layers), "expected only one layer") + if file == "" { + assert.Equal(t, v1.DescriptorEmptyJSON.MediaType, m.Layers[0].MediaType, "did not record empty JSON as layer") + assert.Equal(t, v1.DescriptorEmptyJSON.Digest, m.Layers[0].Digest, "did not record empty JSON as layer") + assert.Equal(t, v1.DescriptorEmptyJSON.Size, m.Layers[0].Size, "did not record empty JSON as layer") + } else { + // we need to have preserved its size + st, err := os.Stat(file) + require.NoError(t, err) + assert.Equal(t, st.Size(), m.Layers[0].Size, "did not record size of file") + // did we set the type correctly? + expectedLayerMediaType := fileMediaType + if layerMediaType != nil { + expectedLayerMediaType = *layerMediaType + } + assert.Equal(t, expectedLayerMediaType, m.Layers[0].MediaType, "recorded MediaType for layer was wrong") + // did we set the digest correctly? + f, err := os.Open(file) + require.NoError(t, err) + defer f.Close() + digester := m.Layers[0].Digest.Algorithm().Digester() + _, err = io.Copy(digester.Hash(), f) + require.NoError(t, err) + assert.Equal(t, digester.Digest().String(), m.Layers[0].Digest.String(), "recorded digest was wrong") + // did we add that annotation? + if excludeTitles && file != "" { + assert.Nil(t, m.Layers[0].Annotations, "expected no layer annotations") + } else { + assert.Equal(t, 1, len(m.Layers[0].Annotations), "expected a layer annotation") + assert.Equal(t, fileBasename, m.Layers[0].Annotations[v1.AnnotationTitle], "expected a title annotation") + } + } + // did we set the annotations? + assert.EqualValues(t, annotations, m.Annotations, "recorded annotations were wrong") + if subjectReference != nil { + // did we set the subject right? + subject, err := subjectReference.NewImageSource(ctx, &types.SystemContext{}) + require.NoError(t, err) + defer subject.Close() + subjectManifestBytes, subjectManifestType, err := subject.GetManifest(ctx, nil) + require.NoError(t, err) + subjectManifestDigest, err := manifest.Digest(subjectManifestBytes) + require.NoError(t, err) + var s v1.Manifest + err = json.Unmarshal(subjectManifestBytes, &s) + require.NoError(t, err) + assert.Equal(t, m.Subject.Digest, subjectManifestDigest) + assert.Equal(t, m.Subject.MediaType, subjectManifestType) + assert.Equal(t, int64(len(subjectManifestBytes)), m.Subject.Size) + } + }) + } + for file, fileMediaType := range map[string]string{ + "": v1.DescriptorEmptyJSON.MediaType, + filepath.Join("..", "testdata", "containers.conf"): "text/plain", + filepath.Join("..", "testdata", "oci-name-only.tar.gz"): "application/gzip", + filepath.Join("..", "..", "logos", "containers.png"): "image/png", + } { + defaultManifestArtifactType := "application/vnd.unknown.artifact.v1" + manifestArtifactType := &defaultManifestArtifactType + defaultManifestMediaType := string(v1.MediaTypeImageManifest) + manifestMediaType := &defaultManifestMediaType + platform := v1.Platform{OS: runtime.GOOS, Architecture: runtime.GOARCH} + configDescriptor := &v1.DescriptorEmptyJSON + configFile := "" + emptyString := "" + layerMediaType := &emptyString + layerArtifactType := &emptyString + annotations := make(map[string]string) + excludeTitles := false + for _, manifestArtifactType := range []string{"(nil)", "", "application/vnd.unknown.artifact.v1"} { + manifestArtifactType := &manifestArtifactType + if *manifestArtifactType == "(nil)" { + manifestArtifactType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + for _, manifestMediaType := range []string{"(nil)", "", v1.MediaTypeImageManifest} { + manifestMediaType := &manifestMediaType + if *manifestMediaType == "(nil)" { + manifestMediaType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + } + for _, platform := range []v1.Platform{ + {}, + { + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + }, + } { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, configDescriptor := range []*v1.Descriptor{ + nil, + {MediaType: v1.MediaTypeImageConfig, Size: 0, Digest: digest.Canonical.FromString("")}, + &v1.DescriptorEmptyJSON, + } { + for _, configFile := range []string{ + "", + configFile, + } { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + } + for _, layerMediaType := range []string{"(nil)", "", "text/plain", "application/octet-stream"} { + layerMediaType := &layerMediaType + if *layerMediaType == "(nil)" { + layerMediaType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, layerArtifactType := range []string{"(nil)", "", "text/plain", "application/octet-stream"} { + layerArtifactType := &layerArtifactType + if *layerArtifactType == "(nil)" { + layerArtifactType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, annotations := range []map[string]string{ + nil, + {}, + { + "annotationA": "valueA", + }, + { + "annotationB": "valueB", + "annotationC": "valueC", + }, + } { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, subjectName := range []string{"", subjectImageName} { + var subjectReference types.ImageReference + if subjectName != "" { + var err error + subjectReference, err = alltransports.ParseImageName(subjectName) + require.NoError(t, err) + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, excludeTitles := range []bool{false, true} { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + } +} + func TestReference(t *testing.T) { if unshare.IsRootless() { t.Skip("Test can only run as root") @@ -187,12 +509,29 @@ func TestReference(t *testing.T) { }() ref, err := alltransports.ParseImageName(otherListImage) - assert.NoError(t, err, "ParseImageName(%q)", otherListImage) + assert.NoErrorf(t, err, "ParseImageName(%q)", otherListImage) list := Create() _, err = list.Add(ctx, ppc64sys, ref, false) assert.NoError(t, err, "list.Add(all=false)") + emptyJSON := filepath.Join(dir, "empty.json") + err = os.WriteFile(emptyJSON, []byte(""), 0o600) + assert.NoError(t, err) + artifactOptions := AddArtifactOptions{ + ConfigFile: emptyJSON, + } + _, err = list.AddArtifact(ctx, &types.SystemContext{}, artifactOptions) + assert.NoErrorf(t, err, "list.AddArtifact(file=%s)", emptyJSON) + artifactOptions = AddArtifactOptions{ + ConfigDescriptor: &v1.DescriptorEmptyJSON, + } + _, err = list.AddArtifact(ctx, &types.SystemContext{}, artifactOptions) + assert.NoError(t, err, "list.AddArtifact(empty descriptor)") + artifactOptions = AddArtifactOptions{} + _, err = list.AddArtifact(ctx, &types.SystemContext{}, artifactOptions) + assert.NoError(t, err, "list.AddArtifact(no descriptor, no file)") + listRef, err := list.Reference(store, cp.CopyAllImages, nil) assert.Error(t, err, "list.Reference(never saved)") assert.Nilf(t, listRef, "list.Reference(never saved)") @@ -287,7 +626,7 @@ func TestPushManifest(t *testing.T) { assert.NoError(t, err, "ParseImageName()") ref, err := alltransports.ParseImageName(otherListImage) - assert.NoError(t, err, "ParseImageName(%q)", otherListImage) + assert.NoErrorf(t, err, "ParseImageName(%q)", otherListImage) list := Create() _, err = list.Add(ctx, sys, ref, true)