diff --git a/libimage/manifests/manifests.go b/libimage/manifests/manifests.go index 130abd3a1..439f77adb 100644 --- a/libimage/manifests/manifests.go +++ b/libimage/manifests/manifests.go @@ -1,15 +1,20 @@ package manifests import ( + "bytes" "context" "encoding/json" "errors" "fmt" "io" + "mime" + "net/http" "os" + "path/filepath" "strings" "time" + "github.com/containers/common/internal" "github.com/containers/common/pkg/manifests" "github.com/containers/common/pkg/retry" "github.com/containers/common/pkg/supplemented" @@ -17,6 +22,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/image" "github.com/containers/image/v5/manifest" + ocilayout "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/pkg/compression" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/signature/signer" @@ -25,8 +31,10 @@ import ( "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" "github.com/containers/storage" + "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/lockfile" digest "github.com/opencontainers/go-digest" + imgspec "github.com/opencontainers/image-spec/specs-go" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" "golang.org/x/exp/slices" @@ -37,8 +45,9 @@ const ( ) const ( - instancesData = "instances.json" - artifactsData = "artifacts.json" + instancesData = "instances.json" + artifactsData = "artifacts.json" + pushingArtifactsSubdirectory = "referenced-artifacts" ) // LookupReferenceFunc return an image reference based on the specified one. @@ -74,6 +83,7 @@ type List interface { Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error) Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error) + AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error) } // PushOptions includes various settings which are needed for pushing the @@ -222,7 +232,7 @@ func (l *list) SaveToImage(store storage.Store, imageID string, names []string, // in the list, or an error if the list has never been saved to a local image. func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) { if l.instances[""] == "" { - return nil, fmt.Errorf("building reference to list: %w", ErrListImageUnknown) + return nil, fmt.Errorf("building reference to list, appears to have not been saved first: %w", ErrListImageUnknown) } s, err := is.Transport.ParseStoreReference(store, l.instances[""]) if err != nil { @@ -246,6 +256,92 @@ func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, in } } } + if len(l.artifacts.Manifests) > 0 { + img, err := is.Transport.GetImage(s) + if err != nil { + return nil, fmt.Errorf("locating image %s: %w", transports.ImageName(s), err) + } + imgDirectory, err := store.ImageDirectory(img.ID) + if err != nil { + return nil, fmt.Errorf("locating per-image directory for %s: %w", img.ID, err) + } + tmp, err := os.MkdirTemp(imgDirectory, pushingArtifactsSubdirectory) + if err != nil { + return nil, err + } + for artifactManifestDigest, contents := range l.artifacts.Manifests { + // create the blobs directory + blobsDir := filepath.Join(tmp, "blobs", artifactManifestDigest.Algorithm().String()) + if err := os.MkdirAll(blobsDir, 0o700); err != nil { + return nil, fmt.Errorf("creating directory for blobs: %w", err) + } + // write the artifact manifest + if err := os.WriteFile(filepath.Join(blobsDir, artifactManifestDigest.Encoded()), []byte(contents), 0o644); err != nil { + return nil, fmt.Errorf("writing artifact manifest as blob: %w", err) + } + // symlink all of the referenced files and write the inlined blobs into the blobs directory + var referencedBlobDigests []digest.Digest + var symlinkedFiles []string + if referencedConfigDigest, ok := l.artifacts.Configs[artifactManifestDigest]; ok { + referencedBlobDigests = append(referencedBlobDigests, referencedConfigDigest) + } + referencedBlobDigests = append(referencedBlobDigests, l.artifacts.Layers[artifactManifestDigest]...) + for _, referencedBlobDigest := range referencedBlobDigests { + referencedFile, knownFile := l.artifacts.Detached[referencedBlobDigest] + referencedBlob, knownBlob := l.artifacts.Blobs[referencedBlobDigest] + if !knownFile && !knownBlob { + return nil, fmt.Errorf(`internal error: no file or blob with artifact "layer" digest %q recorded`, referencedBlobDigest) + } + expectedLayerBlobPath := filepath.Join(blobsDir, referencedBlobDigest.Encoded()) + if knownFile { + if err := os.Symlink(referencedFile, expectedLayerBlobPath); err != nil { + return nil, err + } + symlinkedFiles = append(symlinkedFiles, referencedFile) + } + if knownBlob { + if err := os.WriteFile(expectedLayerBlobPath, referencedBlob, 0o600); err != nil { + return nil, err + } + } + } + // write the index that refers to this one artifact image + tag := "latest" + indexFile := filepath.Join(tmp, "index.json") + index := v1.Index{ + Versioned: imgspec.Versioned{ + SchemaVersion: 2, + }, + MediaType: v1.MediaTypeImageIndex, + Manifests: []v1.Descriptor{{ + MediaType: v1.MediaTypeImageManifest, + Digest: artifactManifestDigest, + Size: int64(len(contents)), + Annotations: map[string]string{ + v1.AnnotationRefName: tag, + }, + }}, + } + indexBytes, err := json.Marshal(&index) + if err != nil { + return nil, fmt.Errorf("encoding image index for OCI layout: %w", err) + } + if err := os.WriteFile(indexFile, indexBytes, 0o644); err != nil { + return nil, fmt.Errorf("writing image index for OCI layout: %w", err) + } + // write the layout file + layoutFile := filepath.Join(tmp, "oci-layout") + if err := os.WriteFile(layoutFile, []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644); err != nil { + return nil, fmt.Errorf("writing oci-layout file: %w", err) + } + // build the reference to this artifact image's oci layout + ref, err := ocilayout.NewReference(tmp, tag) + if err != nil { + return nil, fmt.Errorf("creating ImageReference for artifact with files %q: %w", symlinkedFiles, err) + } + references = append(references, ref) + } + } for _, instance := range whichInstances { imageName := l.instances[instance] ref, err := alltransports.ParseImageName(imageName) @@ -522,6 +618,279 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag return manifestDigest, nil } +// AddArtifactOptions contains options which control the contents of the +// artifact manifest that AddArtifact will create and add to the image index. + +// This should provide for all of the ways to construct a manifest outlined in +// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage +// * no blobs ￫ set ManifestArtifactType +// * blobs, no configuration ￫ set ManifestArtifactType and possibly LayerMediaType, and provide file names +// * blobs and configuration ￫ set ManifestArtifactType, possibly LayerMediaType, and ConfigDescriptor, and provide file names +// +// The older style of describing artifacts: +// * leave ManifestArtifactType blank +// * specify a zero-length application/vnd.oci.image.config.v1+json config blob +// * set LayerMediaType to a custom type +// +// When reading data produced elsewhere, note that newer tooling will produce +// manifests with ArtifactType set. If the manifest's ArtifactType is not set, +// consumers should consult the config descriptor's MediaType. +type AddArtifactOptions struct { + ManifestArtifactType *string // overall type of the artifact manifest. default: "application/vnd.unknown.artifact.v1" + ManifestMediaType *string // default: the OCI image media type + Platform v1.Platform // default: add to the index without platform information + ConfigDescriptor *v1.Descriptor // default: a descriptor for an explicitly empty config blob + ConfigFile string // path to config contents, recorded if ConfigDescriptor.Size != 0 and ConfigDescriptor.Data is not set + LayerMediaType *string // default: mime.TypeByExtension() if basename contains ".", else http.DetectContentType() + LayerArtifactType *string // optional, default is "" + Annotations map[string]string // optional, default is none + SubjectReference types.ImageReference // optional + ExcludeTitles bool // don't add "org.opencontainers.image.title" annotations set to file base names +} + +// AddArtifact creates an artifact manifest describing the specified file or +// files, then adds them to the specified image index. Returns the +// instanceDigest for the artifact manifest. +// The caller could craft the manifest themselves and use Add() to add it to +// the image index and get the same end-result, but this should save them some +// work. +func (l *list) AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error) { + // If we were given a subject, build a descriptor for it first, since + // it might be remote, and anything else we do before looking at it + // might have to get thrown away if we can't get to it for whatever + // reason. + var subject *v1.Descriptor + if options.SubjectReference != nil { + subjectReference, err := options.SubjectReference.NewImageSource(ctx, sys) + if err != nil { + return "", fmt.Errorf("setting up to read manifest and configuration from subject %q: %w", transports.ImageName(options.SubjectReference), err) + } + defer subjectReference.Close() + subjectManifestBytes, subjectManifestType, err := subjectReference.GetManifest(ctx, nil) + if err != nil { + return "", fmt.Errorf("reading manifest from subject %q: %w", transports.ImageName(options.SubjectReference), err) + } + subjectManifestDigest, err := manifest.Digest(subjectManifestBytes) + if err != nil { + return "", fmt.Errorf("digesting manifest of subject %q: %w", transports.ImageName(options.SubjectReference), err) + } + subject = &v1.Descriptor{ + MediaType: subjectManifestType, + Digest: subjectManifestDigest, + Size: int64(len(subjectManifestBytes)), + } + } + + // Build up the layers list piece by piece. + var layers []v1.Descriptor + fileDigests := make(map[string]digest.Digest) + + if len(files) == 0 { + // https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage + // says that we should have at least one layer listed, even if it's just a placeholder + layers = append(layers, v1.DescriptorEmptyJSON) + } + for _, file := range files { + if err := func() error { + // Open the file so that we can digest it. + absFile, err := filepath.Abs(file) + if err != nil { + return fmt.Errorf("converting %q to an absolute path: %w", file, err) + } + + f, err := os.Open(absFile) + if err != nil { + return fmt.Errorf("reading %q to determine its digest: %w", file, err) + } + defer f.Close() + + // Hang on to a copy of the first 512 bytes, but digest the whole thing. + digester := digest.Canonical.Digester() + writeCounter := ioutils.NewWriteCounter(digester.Hash()) + var detectableData bytes.Buffer + _, err = io.CopyN(writeCounter, io.TeeReader(f, &detectableData), 512) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("reading %q to determine its digest: %w", file, err) + } + if err == nil { + if _, err := io.Copy(writeCounter, f); err != nil { + return fmt.Errorf("reading %q to determine its digest: %w", file, err) + } + } + fileDigests[absFile] = digester.Digest() + + // If one wasn't specified, figure out what the MediaType should be. + title := filepath.Base(absFile) + layerMediaType := options.LayerMediaType + if layerMediaType == nil { + if index := strings.LastIndex(title, "."); index != -1 { + // File's basename has an extension, try to use a shortcut. + tmp := mime.TypeByExtension(title[index:]) + if tmp != "" { + layerMediaType = &tmp + } + } + if layerMediaType == nil { + // File's basename has no extension or didn't map to a type, look at the contents we saved. + tmp := http.DetectContentType(detectableData.Bytes()) + layerMediaType = &tmp + } + if layerMediaType != nil { + // Strip off any parameters, since we only want the type name. + if parsedMediaType, _, err := mime.ParseMediaType(*layerMediaType); err == nil { + layerMediaType = &parsedMediaType + } + } + } + + // Only set an ArtifactType for the layer if one was specified. + layerArtifactType := options.LayerArtifactType + if layerArtifactType == nil { + tmp := "" + layerArtifactType = &tmp + } + descriptor := v1.Descriptor{ + MediaType: *layerMediaType, + ArtifactType: *layerArtifactType, + Digest: fileDigests[absFile], + Size: writeCounter.Count, + } + // Add a title annotation unless we were explicitly told not to. + if !options.ExcludeTitles { + descriptor.Annotations = map[string]string{ + v1.AnnotationTitle: title, + } + } + layers = append(layers, descriptor) + return nil + }(); err != nil { + return "", err + } + } + + // Unless we were explicitly told otherwise, default to the normal image MediaType. + mediaType := v1.MediaTypeImageManifest + if options.ManifestMediaType != nil && *options.ManifestMediaType != "" { + mediaType = *options.ManifestMediaType + } + + // Unless we were told what this is, use the default that ORAS uses. + artifactType := "application/vnd.unknown.artifact.v1" + if options.ManifestArtifactType != nil { + artifactType = *options.ManifestArtifactType + } + + // Unless we were explicitly told otherwise, default to an empty config blob. + configDescriptor := internal.DeepCopyDescriptor(&v1.DescriptorEmptyJSON) + if options.ConfigDescriptor != nil { + configDescriptor = internal.DeepCopyDescriptor(options.ConfigDescriptor) + } else if options.ConfigFile != "" { + configDescriptor = &v1.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Digest: "", // to be figured out below + Size: -1, // to be figured out below + } + } + configFilePath := "" + if configDescriptor.Size != 0 { + if len(configDescriptor.Data) == 0 { + if options.ConfigFile == "" { + return "", fmt.Errorf("needed config data file, but none was provided") + } + filePath, err := filepath.Abs(options.ConfigFile) + if err != nil { + return "", fmt.Errorf("recording artifact config data file %q: %w", options.ConfigFile, err) + } + digester := digest.Canonical.Digester() + counter := ioutils.NewWriteCounter(digester.Hash()) + if err := func() error { + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("reading artifact config data file %q: %w", options.ConfigFile, err) + } + defer f.Close() + if _, err := io.Copy(counter, f); err != nil { + return fmt.Errorf("digesting artifact config data file %q: %w", options.ConfigFile, err) + } + return nil + }(); err != nil { + return "", err + } + configDescriptor.Data = nil + configDescriptor.Size = counter.Count + configDescriptor.Digest = digester.Digest() + configFilePath = filePath + } else { + decoder := bytes.NewReader(configDescriptor.Data) + digester := digest.Canonical.Digester() + counter := ioutils.NewWriteCounter(digester.Hash()) + if _, err := io.Copy(counter, decoder); err != nil { + return "", fmt.Errorf("digesting inlined artifact config data: %w", err) + } + configDescriptor.Size = counter.Count + configDescriptor.Digest = digester.Digest() + } + } else { + configDescriptor.Digest = digest.Canonical.FromString("") + } + + // Construct the manifest. + artifactManifest := v1.Manifest{ + Versioned: imgspec.Versioned{ + SchemaVersion: 2, + }, + MediaType: mediaType, + ArtifactType: artifactType, + Config: *configDescriptor, + Layers: layers, + Subject: subject, + } + // Add in annotations, more or less exactly as specified. + if options.Annotations != nil { + artifactManifest.Annotations = make(map[string]string) + } + for k, v := range options.Annotations { + artifactManifest.Annotations[k] = v + } + + // Encode and save the data we care about. + artifactManifestBytes, err := json.Marshal(artifactManifest) + if err != nil { + return "", fmt.Errorf("marshalling the artifact manifest: %w", err) + } + artifactManifestDigest, err := manifest.Digest(artifactManifestBytes) + if err != nil { + return "", fmt.Errorf("digesting the artifact manifest: %w", err) + } + l.artifacts.Manifests[artifactManifestDigest] = string(artifactManifestBytes) + l.artifacts.Layers[artifactManifestDigest] = nil + if configFilePath != "" { + l.artifacts.Configs[artifactManifestDigest] = artifactManifest.Config.Digest + l.artifacts.Detached[artifactManifest.Config.Digest] = configFilePath + l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], configFilePath) + } + if len(artifactManifest.Config.Data) != 0 { + l.artifacts.Configs[artifactManifestDigest] = artifactManifest.Config.Digest + l.artifacts.Blobs[artifactManifest.Config.Digest] = slices.Clone(artifactManifest.Config.Data) + } + for filePath, fileDigest := range fileDigests { + l.artifacts.Layers[artifactManifestDigest] = append(l.artifacts.Layers[artifactManifestDigest], fileDigest) + l.artifacts.Detached[fileDigest] = filePath + l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], filePath) + } + // Add this artifact manifest to the image index. + if err := l.AddInstance(artifactManifestDigest, int64(len(artifactManifestBytes)), artifactManifest.MediaType, options.Platform.OS, options.Platform.Architecture, options.Platform.OSVersion, options.Platform.OSFeatures, options.Platform.Variant, nil, nil); err != nil { + return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err) + } + // Set the artifact type in the image index entry if we have one, since AddInstance() didn't do that for us. + if artifactManifest.ArtifactType != "" { + if err := l.List.SetArtifactType(&artifactManifestDigest, artifactManifest.ArtifactType); err != nil { + return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err) + } + } + return artifactManifestDigest, nil +} + // Remove filters out any instances in the list which match the specified digest. func (l *list) Remove(instanceDigest digest.Digest) error { err := l.List.Remove(instanceDigest) diff --git a/libimage/manifests/manifests_test.go b/libimage/manifests/manifests_test.go index ab20cb511..a2d5d0947 100644 --- a/libimage/manifests/manifests_test.go +++ b/libimage/manifests/manifests_test.go @@ -3,22 +3,32 @@ package manifests import ( "bytes" "context" + "encoding/json" "fmt" + "io" + "os" "path/filepath" + "runtime" + "sort" + "strings" "testing" "time" + "github.com/containerd/containerd/platforms" "github.com/containers/common/pkg/manifests" cp "github.com/containers/image/v5/copy" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/compression" + "github.com/containers/image/v5/transports" "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" "github.com/containers/storage" "github.com/containers/storage/pkg/unshare" digest "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var ( @@ -163,6 +173,319 @@ func TestAddRemove(t *testing.T) { assert.Equalf(t, len(list.Instances()), 1, "too many instances added", otherListInstanceDigest) } +func TestAddArtifact(t *testing.T) { + if unshare.IsRootless() { + t.Skip("Test can only run as root") + } + ctx := context.Background() + dir := t.TempDir() + storeOptions := storage.StoreOptions{ + GraphRoot: filepath.Join(dir, "root"), + RunRoot: filepath.Join(dir, "runroot"), + GraphDriverName: "vfs", + } + emptyConfigFile := filepath.Join(dir, "empty.json") + err := os.WriteFile(emptyConfigFile, []byte("{}"), 0o600) + assert.NoError(t, err, "error creating a mostly-empty file") + store, err := storage.GetStore(storeOptions) + assert.NoError(t, err, "error opening store") + if store == nil { + return + } + defer func() { + if _, err := store.Shutdown(true); err != nil { + assert.NoError(t, err, "error closing store") + } + }() + subjectImageName := "oci-archive:" + filepath.Join("..", "testdata", "oci-name-only.tar.gz") + subjectReference, err := alltransports.ParseImageName(subjectImageName) + require.NoError(t, err) + testCombination := func(t *testing.T, file, fileMediaType string, manifestMediaType, manifestArtifactType *string, platform v1.Platform, configDescriptor *v1.Descriptor, configFile string, layerMediaType, layerArtifactType *string, annotations map[string]string, subjectReference types.ImageReference, excludeTitles bool) { + subjectName := "" + if subjectReference != nil { + subjectName = transports.ImageName(subjectReference) + } + configMediaType := "" + if configDescriptor != nil { + configMediaType = configDescriptor.MediaType + } + var platformStr string + if platform.OS != "" && platform.Architecture != "" { + platformStr = platforms.Format(platform) + } + manifestMediaTypeStr, manifestArtifactTypeStr := "", "" + if manifestMediaType != nil { + manifestMediaTypeStr = *manifestMediaType + } + if manifestArtifactType != nil { + manifestArtifactTypeStr = *manifestArtifactType + } + layerMediaTypeStr, layerArtifactTypeStr := "", "" + if layerMediaType != nil { + layerMediaTypeStr = *layerMediaType + } + if layerArtifactType != nil { + layerArtifactTypeStr = *layerArtifactType + } + annotationsStr := "" + if annotations != nil { + var annotationsSlice []string + for k, v := range annotations { + annotationsSlice = append(annotationsSlice, fmt.Sprintf("%q=%q", k, v)) + } + sort.Strings(annotationsSlice) + annotationsStr = "{" + strings.Join(annotationsSlice, ",") + "}" + } + fileBasename := "" + if file != "" { + fileBasename = filepath.Base(file) + } + desc := fmt.Sprint("file=", fileBasename, ",fileMediaType=", fileMediaType, ",manifestMediaType=", manifestMediaTypeStr, ",manifestArtifactType=", manifestArtifactTypeStr, ",platform=", platformStr, ",configMediaType=", configMediaType, ",configFile=", configFile, ",layerMediaType=", layerMediaTypeStr, ",layerArtifactType=", layerArtifactTypeStr, ",annotations=", annotationsStr, ",subject=", subjectName, ",excludeTitles=", excludeTitles) + t.Run(desc, func(t *testing.T) { + // create the new index and add the file to it + options := AddArtifactOptions{ + ManifestArtifactType: manifestArtifactType, + ManifestMediaType: manifestMediaType, + Platform: platform, + ConfigDescriptor: configDescriptor, + ConfigFile: configFile, + LayerMediaType: layerMediaType, + LayerArtifactType: layerArtifactType, + Annotations: annotations, + SubjectReference: subjectReference, + ExcludeTitles: excludeTitles, + } + list := Create() + var instanceDigest digest.Digest + if file != "" { + instanceDigest, err = list.AddArtifact(ctx, sys, options, file) + } else { + instanceDigest, err = list.AddArtifact(ctx, sys, options) + } + assert.NoErrorf(t, err, "list.AddArtifact(%#v)", options) + assert.Equal(t, 1, len(list.Instances()), "too many instances added") + // have to save it before we can create a reference to it + _, err = list.SaveToImage(store, "", nil, "") + require.NoError(t, err) + // get ready to copy it, sort of + ref, err := list.Reference(store, cp.CopyAllImages, nil) + require.NoError(t, err) + // fetch the manifest for the artifact that we just added to the index + src, err := ref.NewImageSource(ctx, &types.SystemContext{}) + require.NoError(t, err) + defer src.Close() + manifestBytes, manifestType, err := src.GetManifest(ctx, &instanceDigest) + require.NoError(t, err) + // decode the artifact manifest + var m v1.Manifest + if annotations != nil { + m.Annotations = make(map[string]string) + } + err = json.Unmarshal(manifestBytes, &m) + require.NoError(t, err) + // check that the artifact manifest looks right + expectedManifestMediaType := v1.MediaTypeImageManifest + if manifestMediaType != nil && *manifestMediaType != "" { + expectedManifestMediaType = *manifestMediaType + } + assert.Equal(t, expectedManifestMediaType, manifestType) + assert.Equal(t, expectedManifestMediaType, m.MediaType) + expectedManifestArtifactType := "application/vnd.unknown.artifact.v1" + if manifestArtifactType != nil { + expectedManifestArtifactType = *manifestArtifactType + } + assert.Equal(t, expectedManifestArtifactType, m.ArtifactType) + // check the config blob info + configFileSize := v1.DescriptorEmptyJSON.Size + configFileDigest := v1.DescriptorEmptyJSON.Digest + if configFile != "" { + f, err := os.Open(configFile) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, f.Close()) }) + st, err := f.Stat() + require.NoError(t, err) + configFileSize = st.Size() + digester := digest.Canonical.Digester() + _, err = io.Copy(digester.Hash(), f) + require.NoError(t, err) + configFileDigest = digester.Digest() + } + switch { + case configDescriptor != nil && configFile != "": + assert.Equal(t, configDescriptor.MediaType, m.Config.MediaType, "did not record expected media type for config with file") + assert.Equal(t, configFileDigest, m.Config.Digest, "did not record expected digest for config with file") + assert.Equal(t, configFileSize, m.Config.Size, "did not record expected size for config with file") + case configFile != "": + assert.Equal(t, v1.MediaTypeImageConfig, m.Config.MediaType, "did not record expected media type for config file") + assert.Equal(t, configFileDigest, m.Config.Digest, "did not record expected digest for config with file") + assert.Equal(t, configFileSize, m.Config.Size, "did not record expected size for config with file") + case configDescriptor != nil: + assert.Equal(t, configDescriptor.MediaType, m.Config.MediaType, "did not record expected mediaType for empty config") + if false { + d := configDescriptor.Digest + s := configDescriptor.Size + if d.Validate() != nil { + s = 0 + d = digest.Canonical.FromString("") + } + assert.Equal(t, d, m.Config.Digest, "did not record expected digest for empty config") + assert.Equal(t, s, m.Config.Size, "did not record expected digest for empty config") + } + assert.Equal(t, configDescriptor.Digest, m.Config.Digest, "did not record expected digest for empty config") + assert.Equal(t, configDescriptor.Size, m.Config.Size, "did not record expected digest for empty config") + default: + assert.Equal(t, v1.DescriptorEmptyJSON.MediaType, m.Config.MediaType, "did not record expected mediaType for empty config and no config file") + assert.Equal(t, v1.DescriptorEmptyJSON.Digest, m.Config.Digest, "did not record expected digest for empty config and no config file") + assert.Equal(t, v1.DescriptorEmptyJSON.Size, m.Config.Size, "did not record expected digest for empty config and no config file") + } + // if we had a file, it should be there as the "layer", otherwise it should be the empty descriptor + assert.Equal(t, 1, len(m.Layers), "expected only one layer") + if file == "" { + assert.Equal(t, v1.DescriptorEmptyJSON.MediaType, m.Layers[0].MediaType, "did not record empty JSON as layer") + assert.Equal(t, v1.DescriptorEmptyJSON.Digest, m.Layers[0].Digest, "did not record empty JSON as layer") + assert.Equal(t, v1.DescriptorEmptyJSON.Size, m.Layers[0].Size, "did not record empty JSON as layer") + } else { + // we need to have preserved its size + st, err := os.Stat(file) + require.NoError(t, err) + assert.Equal(t, st.Size(), m.Layers[0].Size, "did not record size of file") + // did we set the type correctly? + expectedLayerMediaType := fileMediaType + if layerMediaType != nil { + expectedLayerMediaType = *layerMediaType + } + assert.Equal(t, expectedLayerMediaType, m.Layers[0].MediaType, "recorded MediaType for layer was wrong") + // did we set the digest correctly? + f, err := os.Open(file) + require.NoError(t, err) + defer f.Close() + digester := m.Layers[0].Digest.Algorithm().Digester() + _, err = io.Copy(digester.Hash(), f) + require.NoError(t, err) + assert.Equal(t, digester.Digest().String(), m.Layers[0].Digest.String(), "recorded digest was wrong") + // did we add that annotation? + if excludeTitles && file != "" { + assert.Nil(t, m.Layers[0].Annotations, "expected no layer annotations") + } else { + assert.Equal(t, 1, len(m.Layers[0].Annotations), "expected a layer annotation") + assert.Equal(t, fileBasename, m.Layers[0].Annotations[v1.AnnotationTitle], "expected a title annotation") + } + } + // did we set the annotations? + assert.EqualValues(t, annotations, m.Annotations, "recorded annotations were wrong") + if subjectReference != nil { + // did we set the subject right? + subject, err := subjectReference.NewImageSource(ctx, &types.SystemContext{}) + require.NoError(t, err) + defer subject.Close() + subjectManifestBytes, subjectManifestType, err := subject.GetManifest(ctx, nil) + require.NoError(t, err) + subjectManifestDigest, err := manifest.Digest(subjectManifestBytes) + require.NoError(t, err) + var s v1.Manifest + err = json.Unmarshal(subjectManifestBytes, &s) + require.NoError(t, err) + assert.Equal(t, m.Subject.Digest, subjectManifestDigest) + assert.Equal(t, m.Subject.MediaType, subjectManifestType) + assert.Equal(t, int64(len(subjectManifestBytes)), m.Subject.Size) + } + }) + } + for file, fileMediaType := range map[string]string{ + "": v1.DescriptorEmptyJSON.MediaType, + filepath.Join("..", "testdata", "containers.conf"): "text/plain", + filepath.Join("..", "testdata", "oci-name-only.tar.gz"): "application/gzip", + filepath.Join("..", "..", "logos", "containers.png"): "image/png", + } { + defaultManifestArtifactType := "application/vnd.unknown.artifact.v1" + manifestArtifactType := &defaultManifestArtifactType + defaultManifestMediaType := string(v1.MediaTypeImageManifest) + manifestMediaType := &defaultManifestMediaType + platform := v1.Platform{OS: runtime.GOOS, Architecture: runtime.GOARCH} + configDescriptor := &v1.DescriptorEmptyJSON + configFile := "" + emptyString := "" + layerMediaType := &emptyString + layerArtifactType := &emptyString + annotations := make(map[string]string) + excludeTitles := false + for _, manifestArtifactType := range []string{"(nil)", "", "application/vnd.unknown.artifact.v1"} { + manifestArtifactType := &manifestArtifactType + if *manifestArtifactType == "(nil)" { + manifestArtifactType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + for _, manifestMediaType := range []string{"(nil)", "", v1.MediaTypeImageManifest} { + manifestMediaType := &manifestMediaType + if *manifestMediaType == "(nil)" { + manifestMediaType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + } + for _, platform := range []v1.Platform{ + {}, + { + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + }, + } { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, configDescriptor := range []*v1.Descriptor{ + nil, + {MediaType: v1.MediaTypeImageConfig, Size: 0, Digest: digest.Canonical.FromString("")}, + &v1.DescriptorEmptyJSON, + } { + for _, configFile := range []string{ + "", + configFile, + } { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + } + for _, layerMediaType := range []string{"(nil)", "", "text/plain", "application/octet-stream"} { + layerMediaType := &layerMediaType + if *layerMediaType == "(nil)" { + layerMediaType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, layerArtifactType := range []string{"(nil)", "", "text/plain", "application/octet-stream"} { + layerArtifactType := &layerArtifactType + if *layerArtifactType == "(nil)" { + layerArtifactType = nil + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, annotations := range []map[string]string{ + nil, + {}, + { + "annotationA": "valueA", + }, + { + "annotationB": "valueB", + "annotationC": "valueC", + }, + } { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, subjectName := range []string{"", subjectImageName} { + var subjectReference types.ImageReference + if subjectName != "" { + var err error + subjectReference, err = alltransports.ParseImageName(subjectName) + require.NoError(t, err) + } + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + for _, excludeTitles := range []bool{false, true} { + testCombination(t, file, fileMediaType, manifestMediaType, manifestArtifactType, platform, configDescriptor, configFile, layerMediaType, layerArtifactType, annotations, subjectReference, excludeTitles) + } + } +} + func TestReference(t *testing.T) { if unshare.IsRootless() { t.Skip("Test can only run as root") @@ -187,12 +510,29 @@ func TestReference(t *testing.T) { }() ref, err := alltransports.ParseImageName(otherListImage) - assert.NoError(t, err, "ParseImageName(%q)", otherListImage) + assert.NoErrorf(t, err, "ParseImageName(%q)", otherListImage) list := Create() _, err = list.Add(ctx, ppc64sys, ref, false) assert.NoError(t, err, "list.Add(all=false)") + emptyJSON := filepath.Join(dir, "empty.json") + err = os.WriteFile(emptyJSON, []byte(""), 0o600) + assert.NoError(t, err) + artifactOptions := AddArtifactOptions{ + ConfigFile: emptyJSON, + } + _, err = list.AddArtifact(ctx, &types.SystemContext{}, artifactOptions) + assert.NoErrorf(t, err, "list.AddArtifact(file=%s)", emptyJSON) + artifactOptions = AddArtifactOptions{ + ConfigDescriptor: &v1.DescriptorEmptyJSON, + } + _, err = list.AddArtifact(ctx, &types.SystemContext{}, artifactOptions) + assert.NoError(t, err, "list.AddArtifact(empty descriptor)") + artifactOptions = AddArtifactOptions{} + _, err = list.AddArtifact(ctx, &types.SystemContext{}, artifactOptions) + assert.NoError(t, err, "list.AddArtifact(no descriptor, no file)") + listRef, err := list.Reference(store, cp.CopyAllImages, nil) assert.Error(t, err, "list.Reference(never saved)") assert.Nilf(t, listRef, "list.Reference(never saved)") @@ -287,7 +627,7 @@ func TestPushManifest(t *testing.T) { assert.NoError(t, err, "ParseImageName()") ref, err := alltransports.ParseImageName(otherListImage) - assert.NoError(t, err, "ParseImageName(%q)", otherListImage) + assert.NoErrorf(t, err, "ParseImageName(%q)", otherListImage) list := Create() _, err = list.Add(ctx, sys, ref, true)