Skip to content

Commit

Permalink
libimage/manifests.list: hold artifact manifests
Browse files Browse the repository at this point in the history
Add an AddArtifact() method which will craft an artifact manifest which
references one or more files and then add that manifest to the index.

When we need to build a reference to a list that includes our artifact
manifests, save the manifests and symlinks to their "layer" blobs in a
per-image directory, and add references to those locations to the list
of images we can search for manifests and blobs.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
  • Loading branch information
nalind committed Jan 30, 2024
1 parent ff955f3 commit 94eac93
Show file tree
Hide file tree
Showing 2 changed files with 588 additions and 3 deletions.
308 changes: 307 additions & 1 deletion libimage/manifests/manifests.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
package manifests

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"time"

Expand All @@ -16,6 +21,7 @@ import (
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/manifest"
ocilayout "github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/pkg/compression"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/signature/signer"
Expand All @@ -24,8 +30,10 @@ import (
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
"github.com/containers/storage/pkg/ioutils"
"github.com/containers/storage/pkg/lockfile"
digest "github.com/opencontainers/go-digest"
imgspec "github.com/opencontainers/image-spec/specs-go"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
Expand Down Expand Up @@ -71,6 +79,7 @@ type List interface {
Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error)
Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error)
Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error)
AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error)
}

// PushOptions includes various settings which are needed for pushing the
Expand Down Expand Up @@ -212,7 +221,7 @@ func (l *list) SaveToImage(store storage.Store, imageID string, names []string,
// in the list, or an error if the list has never been saved to a local image.
func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) {
if l.instances[""] == "" {
return nil, fmt.Errorf("building reference to list: %w", ErrListImageUnknown)
return nil, fmt.Errorf("building reference to list, appears to have not been saved first: %w", ErrListImageUnknown)
}
s, err := is.Transport.ParseStoreReference(store, l.instances[""])
if err != nil {
Expand All @@ -236,6 +245,87 @@ func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, in
}
}
}
if len(l.artifacts.Manifests) > 0 {
img, err := is.Transport.GetImage(s)
if err != nil {
return nil, fmt.Errorf("locating image %s: %w", transports.ImageName(s), err)
}
imgDirectory, err := store.ImageDirectory(img.ID)
if err != nil {
return nil, fmt.Errorf("locating per-image directory for %s: %w", img.ID, err)
}
tmp, err := os.MkdirTemp(imgDirectory, "referenced-artifacts")
if err != nil {
return nil, err
}
for artifactManifestDigest, contents := range l.artifacts.Manifests {
// write the empty blob, in case we need it instead of an inlined copy
emptyBlobDir := filepath.Join(tmp, "blobs", "sha256")
if err := os.MkdirAll(emptyBlobDir, 0o700); err != nil {
return nil, fmt.Errorf("creating directory for the empty blob: %w", err)
}
if err := os.WriteFile(filepath.Join(emptyBlobDir, "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"), []byte("{}"), 0o644); err != nil {
return nil, fmt.Errorf("writing the empty blob: %w", err)
}
// create the main blobs directory, in case it's different from the one we just created
blobsDir := filepath.Join(tmp, "blobs", artifactManifestDigest.Algorithm().String())
if err := os.MkdirAll(blobsDir, 0o700); err != nil {
return nil, fmt.Errorf("creating directory for blobs: %w", err)
}
// write the artifact manifest
if err := os.WriteFile(filepath.Join(blobsDir, artifactManifestDigest.Encoded()), []byte(contents), 0o644); err != nil {
return nil, fmt.Errorf("writing artifact manifest as blob: %w", err)
}
// symlink all of the referenced files into the blobs directory
var files []string
for _, referencedFileDigest := range l.artifacts.Layers[artifactManifestDigest] {
referencedFile, known := l.artifacts.Detached[referencedFileDigest]
if !known {
return nil, fmt.Errorf(`internal error: no file with artifact "layer" digest %q recorded`, referencedFileDigest)
}
files = append(files, referencedFile)
expectedLayerBlobPath := filepath.Join(blobsDir, referencedFileDigest.Encoded())
if err := os.Symlink(referencedFile, expectedLayerBlobPath); err != nil {
return nil, err
}
}
// write the index that refers to this one artifact image
tag := "latest"
indexFile := filepath.Join(tmp, "index.json")
index := v1.Index{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
MediaType: v1.MediaTypeImageIndex,
Manifests: []v1.Descriptor{{
MediaType: v1.MediaTypeImageManifest,
Digest: artifactManifestDigest,
Size: int64(len(contents)),
Annotations: map[string]string{
v1.AnnotationRefName: tag,
},
}},
}
indexBytes, err := json.Marshal(&index)
if err != nil {
return nil, fmt.Errorf("encoding image index for OCI layout: %w", err)
}
if err := os.WriteFile(indexFile, indexBytes, 0o644); err != nil {
return nil, fmt.Errorf("writing image index for OCI layout: %w", err)
}
// write the layout file
layoutFile := filepath.Join(tmp, "oci-layout")
if err := os.WriteFile(layoutFile, []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644); err != nil {
return nil, fmt.Errorf("writing oci-layout file: %w", err)
}
// build the reference to this artifact image's oci layout
ref, err := ocilayout.NewReference(tmp, tag)
if err != nil {
return nil, fmt.Errorf("creating ImageReference for artifact with files %q: %w", files, err)
}
references = append(references, ref)
}
}
for _, instance := range whichInstances {
imageName := l.instances[instance]
ref, err := alltransports.ParseImageName(imageName)
Expand Down Expand Up @@ -510,6 +600,222 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag
return manifestDigest, nil
}

// AddArtifactOptions contains options which control the contents of the
// artifact manifest that AddArtifact will create and add to the image index.

// This should provide for all of the ways to construct a manifest outlined in
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage
// * no blobs → set ManifestArtifactType
// * blobs, no configuration → set ManifestArtifactType and possibly LayerMediaType, and provide file names
// * blobs and configuration → set ManifestArtifactType, possibly LayerMediaType, and ConfigDescriptor, and provide file names
//
// The older style of describing artifacts:
// * leave ManifestArtifactType blank
// * specify a zero-length application/vnd.oci.image.config.v1+json config blob
// * set LayerMediaType to a custom type
//
// When reading data produced elsewhere, note that newer tooling will produce
// manifests with ArtifactType set. If the manifest's ArtifactType is not set,
// consumers should consult the config descriptor's MediaType.
type AddArtifactOptions struct {
ManifestArtifactType *string // overall type of the artifact manifest. default: "application/vnd.unknown.artifact.v1"
ManifestMediaType *string // default: the OCI image media type
Platform v1.Platform // default: add to the index without platform information
ConfigDescriptor *v1.Descriptor // default: a descriptor for an explicitly empty config blob
LayerMediaType *string // default: mime.TypeByExtension() if basename contains ".", else http.DetectContentType()
LayerArtifactType *string // optional, default is ""
Annotations map[string]string // optional, default is none
SubjectReference types.ImageReference // optional
ExcludeTitles bool // don't add "org.opencontainers.image.title" annotations set to file base names
}

// AddArtifact creates an artifact manifest describing the specified file or
// files, then adds them to the specified image index. Returns the
// instanceDigest for the artifact manifest.
// The caller could craft the manifest themselves and use Add() to add it to
// the image index and get the same end-result, but this should save them some
// work.
func (l *list) AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error) {
// If we were given a subject, build a descriptor for it first, since
// it might be remote, and anything else we do before looking at it
// might have to get thrown away if we can't get to it for whatever
// reason.
var subject *v1.Descriptor
if options.SubjectReference != nil {
subjectReference, err := options.SubjectReference.NewImageSource(ctx, sys)
if err != nil {
return "", fmt.Errorf("setting up to read manifest and configuration from subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
defer subjectReference.Close()
subjectManifestBytes, subjectManifestType, err := subjectReference.GetManifest(ctx, nil)
if err != nil {
return "", fmt.Errorf("reading manifest from subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
subjectManifestDigest, err := manifest.Digest(subjectManifestBytes)
if err != nil {
return "", fmt.Errorf("digesting manifest of subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
subject = &v1.Descriptor{
MediaType: subjectManifestType,
Digest: subjectManifestDigest,
Size: int64(len(subjectManifestBytes)),
}
l.instances[subjectManifestDigest] = transports.ImageName(options.SubjectReference)
}

// Build up the layers list piece by piece.
var layers []v1.Descriptor
fileDigests := make(map[string]digest.Digest)

if len(files) == 0 {
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage
// says that we should have at least one layer listed, even if it's just a placeholder
layers = append(layers, v1.DescriptorEmptyJSON)
}
for _, file := range files {
if err := func() error {
// Open the file so that we can digest it.
absFile, err := filepath.Abs(file)
if err != nil {
return fmt.Errorf("converting %q to an absolute path: %w", file, err)
}

f, err := os.Open(absFile)
if err != nil {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
defer f.Close()

// Hang on to a copy of the first 512 bytes, but digest the whole thing.
digester := digest.Canonical.Digester()
writeCounter := ioutils.NewWriteCounter(digester.Hash())
var detectableData bytes.Buffer
_, err = io.CopyN(writeCounter, io.TeeReader(f, &detectableData), 512)
if err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
if err == nil {
if _, err := io.Copy(writeCounter, f); err != nil {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
}
fileDigests[absFile] = digester.Digest()

// If one wasn't specified, figure out what the MediaType should be.
title := filepath.Base(absFile)
layerMediaType := options.LayerMediaType
if layerMediaType == nil {
if index := strings.LastIndex(title, "."); index != -1 {
// File's basename has an extension, try to use a shortcut.
tmp := mime.TypeByExtension(title[index:])
if tmp != "" {
layerMediaType = &tmp
}
}
if layerMediaType == nil {
// File's basename has no extension or didn't map to a type, look at the contents we saved.
tmp := http.DetectContentType(detectableData.Bytes())
layerMediaType = &tmp
}
if layerMediaType != nil {
// Strip off any parameters, since we only want the type name.
if parsedMediaType, _, err := mime.ParseMediaType(*layerMediaType); err == nil {
layerMediaType = &parsedMediaType
}
}
}

// Only set an ArtifactType for the layer if one was specified.
layerArtifactType := options.LayerArtifactType
if layerArtifactType == nil {
tmp := ""
layerArtifactType = &tmp
}
descriptor := v1.Descriptor{
MediaType: *layerMediaType,
ArtifactType: *layerArtifactType,
Digest: fileDigests[absFile],
Size: writeCounter.Count,
}
// Add a title annotation unless we were explicitly told not to.
if !options.ExcludeTitles {
descriptor.Annotations = map[string]string{
v1.AnnotationTitle: title,
}
}
layers = append(layers, descriptor)
return nil
}(); err != nil {
return "", err
}
}

// Unless we were explicitly told otherwise, default to the normal image MediaType.
mediaType := v1.MediaTypeImageManifest
if options.ManifestMediaType != nil && *options.ManifestMediaType != "" {
mediaType = *options.ManifestMediaType
}

// Unless we were told what this is, use the default that ORAS uses.
artifactType := "application/vnd.unknown.artifact.v1"
if options.ManifestArtifactType != nil {
artifactType = *options.ManifestArtifactType
}

// Unless we were explicitly told otherwise, default to an empty config blob.
configDescriptor := &v1.DescriptorEmptyJSON
if options.ConfigDescriptor != nil {
configDescriptor = options.ConfigDescriptor
}

// Construct the manifest.
artifactManifest := v1.Manifest{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
MediaType: mediaType,
ArtifactType: artifactType,
Config: *configDescriptor,
Layers: layers,
Subject: subject,
}
// Add in annotations, more or less exactly as specified.
if options.Annotations != nil {
artifactManifest.Annotations = make(map[string]string)
}
for k, v := range options.Annotations {
artifactManifest.Annotations[k] = v
}

// Encode and save the data we care about.
artifactManifestBytes, err := json.Marshal(artifactManifest)
if err != nil {
return "", fmt.Errorf("marshalling the artifact manifest: %w", err)
}
artifactManifestDigest, err := manifest.Digest(artifactManifestBytes)
if err != nil {
return "", fmt.Errorf("digesting the artifact manifest: %w", err)
}
l.artifacts.Manifests[artifactManifestDigest] = string(artifactManifestBytes)
l.artifacts.Layers[artifactManifestDigest] = nil
for filePath, fileDigest := range fileDigests {
l.artifacts.Layers[artifactManifestDigest] = append(l.artifacts.Layers[artifactManifestDigest], fileDigest)
l.artifacts.Detached[fileDigest] = filePath
l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], filePath)
}
// Add this artifact manifest to the image index.
if err := l.AddInstance(artifactManifestDigest, int64(len(artifactManifestBytes)), artifactManifest.MediaType, options.Platform.OS, options.Platform.Architecture, options.Platform.OSVersion, options.Platform.OSFeatures, options.Platform.Variant, nil, nil); err != nil {
return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err)
}
// Set the artifact type in the image index entry if we have one, since AddInstance() didn't do that for us.
if artifactManifest.ArtifactType != "" {
if err := l.List.SetArtifactType(&artifactManifestDigest, artifactManifest.ArtifactType); err != nil {
return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err)
}
}
return artifactManifestDigest, nil
}

// Remove filters out any instances in the list which match the specified digest.
func (l *list) Remove(instanceDigest digest.Digest) error {
err := l.List.Remove(instanceDigest)
Expand Down
Loading

0 comments on commit 94eac93

Please sign in to comment.