diff --git a/cmd/hauler/cli/store/save.go b/cmd/hauler/cli/store/save.go index 0d6d082e..873d487b 100644 --- a/cmd/hauler/cli/store/save.go +++ b/cmd/hauler/cli/store/save.go @@ -1,13 +1,25 @@ package store import ( + "bytes" "context" + "encoding/json" "os" + "path" "path/filepath" + "slices" + referencev3 "github.com/distribution/distribution/v3/reference" + "github.com/google/go-containerregistry/pkg/name" + libv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/mholt/archiver/v3" + imagev1 "github.com/opencontainers/image-spec/specs-go/v1" "hauler.dev/go/hauler/internal/flags" + "hauler.dev/go/hauler/pkg/consts" "hauler.dev/go/hauler/pkg/log" ) @@ -34,6 +46,10 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, outputFile string) error { return err } + if err := writeExportsManifest(ctx, "."); err != nil { + return err + } + err = a.Archive([]string{"."}, absOutputfile) if err != nil { return err @@ -42,3 +58,154 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, outputFile string) error { l.Infof("saved store [%s] -> [%s]", o.StoreDir, absOutputfile) return nil } + +type exports struct { + digests []string + records map[string]tarball.Descriptor +} + +func writeExportsManifest(ctx context.Context, dir string) error { + l := log.FromContext(ctx) + + oci, err := layout.FromPath(dir) + if err != nil { + return err + } + + idx, err := oci.ImageIndex() + if err != nil { + return err + } + + imx, err := idx.IndexManifest() + if err != nil { + return err + } + + x := &exports{ + digests: []string{}, + records: map[string]tarball.Descriptor{}, + } + + for _, desc := range imx.Manifests { + l.Debugf("descriptor [%s] >>> %s", desc.Digest.String(), desc.MediaType) + if artifactType := types.MediaType(desc.ArtifactType); artifactType != "" && !artifactType.IsImage() && !artifactType.IsIndex() { + l.Debugf("descriptor [%s] <<< SKIPPING ARTIFACT (%q)", desc.Digest.String(), desc.ArtifactType) + continue + } + if desc.Annotations != nil { + // we only care about images that cosign has added to the layout index + if kind, hasKind := desc.Annotations[consts.KindAnnotationName]; hasKind { + if refName, hasRefName := desc.Annotations[imagev1.AnnotationRefName]; hasRefName { + // branch on image (aka image manifest) or image index + switch kind { + case consts.KindAnnotationImage: + if err := x.record(ctx, idx, desc, refName); err != nil { + return err + } + case consts.KindAnnotationIndex: + l.Debugf("index [%s]: digest=%s, type=%s, size=%d", refName, desc.Digest.String(), desc.MediaType, desc.Size) + iix, err := idx.ImageIndex(desc.Digest) + if err != nil { + return err + } + ixm, err := iix.IndexManifest() + if err != nil { + return err + } + for _, ixd := range ixm.Manifests { + if ixd.MediaType.IsImage() { + if err := x.record(ctx, iix, ixd, refName); err != nil { + return err + } + } + } + default: + l.Debugf("descriptor [%s] <<< SKIPPING KIND (%q)", desc.Digest.String(), kind) + } + } + } + } + } + + buf := bytes.Buffer{} + mnf := x.describe() + err = json.NewEncoder(&buf).Encode(mnf) + if err != nil { + return err + } + + return oci.WriteFile("manifest.json", buf.Bytes(), 0666) +} + +func (x *exports) describe() tarball.Manifest { + m := make(tarball.Manifest, len(x.digests)) + for i, d := range x.digests { + m[i] = x.records[d] + } + return m +} + +func (x *exports) record(ctx context.Context, index libv1.ImageIndex, desc libv1.Descriptor, refname string) error { + l := log.FromContext(ctx) + + digest := desc.Digest.String() + image, err := index.Image(desc.Digest) + if err != nil { + return err + } + + config, err := image.ConfigName() + if err != nil { + return err + } + + xd, recorded := x.records[digest] + if !recorded { + // record one export record per digest + x.digests = append(x.digests, digest) + xd = tarball.Descriptor{ + Config: path.Join(imagev1.ImageBlobsDir, config.Algorithm, config.Hex), + RepoTags: []string{}, + Layers: []string{}, + } + + layers, err := image.Layers() + if err != nil { + return err + } + for _, layer := range layers { + xl, err := layer.Digest() + if err != nil { + return err + } + xd.Layers = append(xd.Layers[:], path.Join(imagev1.ImageBlobsDir, xl.Algorithm, xl.Hex)) + } + } + + ref, err := name.ParseReference(refname) + if err != nil { + return err + } + + // record tags for the digest, eliminating dupes + switch tag := ref.(type) { + case name.Tag: + named, err := referencev3.ParseNormalizedNamed(refname) + if err != nil { + return err + } + named = referencev3.TagNameOnly(named) + repotag := referencev3.FamiliarString(named) + xd.RepoTags = append(xd.RepoTags[:], repotag) + slices.Sort(xd.RepoTags) + xd.RepoTags = slices.Compact(xd.RepoTags) + ref = tag.Digest(digest) + } + + l.Debugf("image [%s]: type=%s, size=%d", ref.Name(), desc.MediaType, desc.Size) + // record export descriptor for the digest + x.records[digest] = xd + + return nil +} diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 579ed4e6..9e4f23ba 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -47,8 +47,9 @@ const ( HaulerVendorPrefix = "vnd.hauler" OCIImageIndexFile = "index.json" - KindAnnotationName = "kind" - KindAnnotation = "dev.cosignproject.cosign/image" + KindAnnotationName = "kind" + KindAnnotationImage = "dev.cosignproject.cosign/image" + KindAnnotationIndex = "dev.cosignproject.cosign/imageIndex" CarbideRegistry = "rgcrprod.azurecr.us" ImageAnnotationKey = "hauler.dev/key" diff --git a/pkg/content/oci.go b/pkg/content/oci.go index 99ae5862..092d2d5f 100644 --- a/pkg/content/oci.go +++ b/pkg/content/oci.go @@ -124,9 +124,9 @@ func (o *OCI) SaveIndex() error { kindJ := descs[j].Annotations["kind"] // Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top. - if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) { + if strings.HasPrefix(kindI, consts.KindAnnotationImage) && !strings.HasPrefix(kindJ, consts.KindAnnotationImage) { return true - } else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) { + } else if !strings.HasPrefix(kindI, consts.KindAnnotationImage) && strings.HasPrefix(kindJ, consts.KindAnnotationImage) { return false } return false // Default: maintain the order. diff --git a/pkg/store/store.go b/pkg/store/store.go index d1b5dcc2..1e7ea914 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -118,7 +118,7 @@ func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (oci Digest: digest.FromBytes(mdata), Size: int64(len(mdata)), Annotations: map[string]string{ - consts.KindAnnotationName: consts.KindAnnotation, + consts.KindAnnotationName: consts.KindAnnotationImage, ocispec.AnnotationRefName: ref, }, URLs: nil,