From e21467faaa9aa4ab512fdb96ebd5faec54f01a15 Mon Sep 17 00:00:00 2001 From: Jacob Blain Christen Date: Fri, 13 Sep 2024 01:58:27 -0700 Subject: [PATCH] enable docker load for hauler tarballs - fixes #276 Signed-off-by: Jacob Blain Christen --- cmd/hauler/cli/store/add.go | 158 +++++++++++++++++++++++++++++++++++- pkg/consts/consts.go | 5 +- pkg/content/oci.go | 4 +- pkg/store/store.go | 2 +- 4 files changed, 163 insertions(+), 6 deletions(-) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index dab32b86..f0d3d386 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -2,14 +2,26 @@ package store import ( "context" + "encoding/json" + "os" + "path" + "path/filepath" + "slices" + "sort" + referencev3 "github.com/distribution/distribution/v3/reference" "github.com/google/go-containerregistry/pkg/name" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" + libv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/tarball" + imagev1 "github.com/opencontainers/image-spec/specs-go/v1" "helm.sh/helm/v3/pkg/action" "hauler.dev/go/hauler/internal/flags" "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1alpha1" "hauler.dev/go/hauler/pkg/artifacts/file" + "hauler.dev/go/hauler/pkg/artifacts/file/getter" + "hauler.dev/go/hauler/pkg/consts" "hauler.dev/go/hauler/pkg/content/chart" "hauler.dev/go/hauler/pkg/cosign" "hauler.dev/go/hauler/pkg/log" @@ -80,12 +92,156 @@ func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform return nil } + p, err := libv1.ParsePlatform(platform) + if err != nil { + return err + } + err = cosign.SaveImage(ctx, s, r.Name(), platform) if err != nil { l.Warnf("unable to add 'image' [%s] to store. skipping...", r.Name()) return nil } + l.Debugf("adding 'image' [%s] to the export manifest", i.Name) + + var ( + exports = make(tarball.Manifest, 0) + records = make(map[string]*tarball.Descriptor) + ) + + exportDescriptor := func(refname string, desc libv1.Descriptor, index libv1.ImageIndex) error { + digest := desc.Digest.String() + image, err := index.Image(desc.Digest) + if err != nil { + return err + } + + config, err := image.ConfigFile() + if err != nil { + return err + } + + cfg, err := image.ConfigName() + if err != nil { + return err + } + + if config != nil && config.Platform() != nil && config.Platform().Satisfies(*p) { + descriptor, recorded := records[digest] + if !recorded { + exports = append(exports, tarball.Descriptor{ + Config: path.Join("blobs", cfg.Algorithm, cfg.Hex), + RepoTags: []string{}, + Layers: []string{}, + }) + descriptor = &exports[len(exports)-1] + layers, err := image.Layers() + if err != nil { + return err + } + for _, layer := range layers { + ldg, err := layer.Digest() + if err != nil { + return err + } + descriptor.Layers = append(descriptor.Layers[:], path.Join("blobs", ldg.Algorithm, ldg.Hex)) + } + } + + ref, err := name.ParseReference(refname) + if err != nil { + return err + } + + switch tag := ref.(type) { + case name.Tag: + ref = tag.Digest(digest) + named, err := referencev3.ParseNormalizedNamed(refname) + if err != nil { + return err + } + named = referencev3.TagNameOnly(named) + repotag := referencev3.FamiliarString(named) + if !slices.Contains(descriptor.RepoTags, repotag) { + descriptor.RepoTags = append(descriptor.RepoTags[:], repotag) + sort.Strings(descriptor.RepoTags) + } + } + records[digest] = descriptor + l.Debugf("image [%s]: type=%s, size=%d", ref.Name(), desc.MediaType, desc.Size) + } + return nil + } + + oci, err := layout.FromPath(s.Root) + if err != nil { + return err + } + + idx, err := oci.ImageIndex() + if err != nil { + return err + } + + imf, err := idx.IndexManifest() + if err != nil { + return err + } + + for _, imd := range imf.Manifests { + l.Debugf("descriptor [%s] >>> %s", imd.Digest.String(), imd.MediaType) + if imd.ArtifactType != "" { + l.Debugf("descriptor [%s] <<< SKIPPING ARTIFACT %q", imd.Digest.String(), imd.ArtifactType) + continue + } + if imd.Annotations != nil { + // we only care about images that cosign has added to the layout index + if kind, hasKind := imd.Annotations[consts.KindAnnotationName]; hasKind { + if refName, hasRefName := imd.Annotations[imagev1.AnnotationRefName]; hasRefName { + // branch on image (aka image manifest) or image index + switch kind { + case consts.KindAnnotationImage: + if err := exportDescriptor(refName, imd, idx); err != nil { + return err + } + case consts.KindAnnotationIndex: + l.Debugf("index [%s]: digest=%s, type=%s, size=%d", refName, imd.Digest.String(), imd.MediaType, imd.Size) + ixi, err := idx.ImageIndex(imd.Digest) + if err != nil { + return err + } + ixm, err := ixi.IndexManifest() + if err != nil { + return err + } + for _, iid := range ixm.Manifests { + if iid.MediaType.IsImage() { + if err := exportDescriptor(refName, iid, ixi); err != nil { + return err + } + } + } + default: + l.Debugf("descriptor [%s] <<< SKIPPING KIND => %q", imd.Digest.String(), kind) + } + } + } + } + } + + manifest, err := os.Create(filepath.Join(s.Root, "manifest.json")) + if err != nil { + return err + } + defer manifest.Close() + defer manifest.Sync() + + err = json.NewEncoder(manifest).Encode(exports) + if err != nil { + return err + } + l.Infof("successfully added 'image' [%s]", r.Name()) return nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 579ed4e6..9e4f23ba 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -47,8 +47,9 @@ const ( HaulerVendorPrefix = "vnd.hauler" OCIImageIndexFile = "index.json" - KindAnnotationName = "kind" - KindAnnotation = "dev.cosignproject.cosign/image" + KindAnnotationName = "kind" + KindAnnotationImage = "dev.cosignproject.cosign/image" + KindAnnotationIndex = "dev.cosignproject.cosign/imageIndex" CarbideRegistry = "rgcrprod.azurecr.us" ImageAnnotationKey = "hauler.dev/key" diff --git a/pkg/content/oci.go b/pkg/content/oci.go index 99ae5862..092d2d5f 100644 --- a/pkg/content/oci.go +++ b/pkg/content/oci.go @@ -124,9 +124,9 @@ func (o *OCI) SaveIndex() error { kindJ := descs[j].Annotations["kind"] // Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top. - if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) { + if strings.HasPrefix(kindI, consts.KindAnnotationImage) && !strings.HasPrefix(kindJ, consts.KindAnnotationImage) { return true - } else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) { + } else if !strings.HasPrefix(kindI, consts.KindAnnotationImage) && strings.HasPrefix(kindJ, consts.KindAnnotationImage) { return false } return false // Default: maintain the order. diff --git a/pkg/store/store.go b/pkg/store/store.go index d1b5dcc2..1e7ea914 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -118,7 +118,7 @@ func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (oci Digest: digest.FromBytes(mdata), Size: int64(len(mdata)), Annotations: map[string]string{ - consts.KindAnnotationName: consts.KindAnnotation, + consts.KindAnnotationName: consts.KindAnnotationImage, ocispec.AnnotationRefName: ref, }, URLs: nil,