From 19ff6fbe5be3a2af579a9a315351b5a4a4f31cca Mon Sep 17 00:00:00 2001 From: Ben Oukhanov Date: Wed, 28 Feb 2024 09:18:55 +0000 Subject: [PATCH] feat: build and push image index Push an image index to have multiple image manifests, instead of single image manifest when needed. Jira-Url: https://issues.redhat.com/browse/CNV-38597 Signed-off-by: Ben Oukhanov --- artifacts/centos/centos.go | 1 + artifacts/centos/centos_test.go | 12 +- artifacts/centosstream/centos-stream.go | 1 + artifacts/centosstream/centos-stream_test.go | 2 + artifacts/fedora/fedora.go | 5 +- artifacts/fedora/fedora_test.go | 6 +- artifacts/ubuntu/ubuntu.go | 7 +- artifacts/ubuntu/ubuntu_test.go | 5 +- cmd/medius/common/registry.go | 98 ++++++++------- cmd/medius/docs/docs.go | 8 +- cmd/medius/images/common.go | 5 +- cmd/medius/images/promote.go | 5 +- cmd/medius/images/push.go | 125 +++++++++++++++---- cmd/medius/images/verify.go | 5 +- pkg/api/artifact.go | 4 +- pkg/build/build.go | 34 ++++- pkg/repository/repository.go | 18 ++- 17 files changed, 244 insertions(+), 97 deletions(-) diff --git a/artifacts/centos/centos.go b/artifacts/centos/centos.go index f6f45020..74985120 100644 --- a/artifacts/centos/centos.go +++ b/artifacts/centos/centos.go @@ -64,6 +64,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) { SHA256Sum: checksum, DownloadURL: baseURL + candidate, AdditionalUniqueTags: getAdditionalTags(c.Version, c.Variant, candidate), + ImageArchitecture: "amd64", }, nil } diff --git a/artifacts/centos/centos_test.go b/artifacts/centos/centos_test.go index fd8c01f1..66c4a54f 100644 --- a/artifacts/centos/centos_test.go +++ b/artifacts/centos/centos_test.go @@ -28,6 +28,7 @@ var _ = Describe("Centos", func() { DownloadURL: "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2", Compression: "", AdditionalUniqueTags: []string{"8.4.2105-20210603.0", "8.4.2105"}, + ImageArchitecture: "amd64", }, map[string]string{ "TEST_ENV_VAR": "test-value", @@ -50,6 +51,7 @@ var _ = Describe("Centos", func() { DownloadURL: "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", Compression: "", AdditionalUniqueTags: []string{"8.3.2011-20201204.2", "8.3.2011"}, + ImageArchitecture: "amd64", }, map[string]string{ "TEST_ENV_VAR": "test-value", @@ -68,8 +70,9 @@ var _ = Describe("Centos", func() { ), Entry("centos:7-2009", "7-2009", "testdata/centos7.checksum", &api.ArtifactDetails{ - SHA256Sum: "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3", - DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2", + SHA256Sum: "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3", + DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2", + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", @@ -90,8 +93,9 @@ var _ = Describe("Centos", func() { ), Entry("centos:7-1809", "7-1809", "testdata/centos7.checksum", &api.ArtifactDetails{ - SHA256Sum: "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2", - DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2", + SHA256Sum: "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2", + DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2", + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", diff --git a/artifacts/centosstream/centos-stream.go b/artifacts/centosstream/centos-stream.go index 34a3ed81..494d276f 100644 --- a/artifacts/centosstream/centos-stream.go +++ b/artifacts/centosstream/centos-stream.go @@ -89,6 +89,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) { SHA256Sum: checksum, DownloadURL: baseURL + candidate, AdditionalUniqueTags: additionalTags, + ImageArchitecture: "amd64", }, nil } diff --git a/artifacts/centosstream/centos-stream_test.go b/artifacts/centosstream/centos-stream_test.go index a1decf7f..03cb9ef3 100644 --- a/artifacts/centosstream/centos-stream_test.go +++ b/artifacts/centosstream/centos-stream_test.go @@ -28,6 +28,7 @@ var _ = Describe("CentosStream", func() { SHA256Sum: "8e22e67687b81e38c7212fc30c47cb24cbc4935c0f2459ed139f498397d1e7cd", DownloadURL: "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210603.0.x86_64.qcow2", AdditionalUniqueTags: []string{"8-20210603.0"}, + ImageArchitecture: "amd64", }, &docs.UserData{ Username: "centos", @@ -54,6 +55,7 @@ var _ = Describe("CentosStream", func() { SHA256Sum: "bcebdc00511d6e18782732570056cfbc7cba318302748bfc8f66be9c0db68142", DownloadURL: "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20211222.0.x86_64.qcow2", AdditionalUniqueTags: []string{"9-20211222.0"}, + ImageArchitecture: "amd64", }, &docs.UserData{ Username: "cloud-user", diff --git a/artifacts/fedora/fedora.go b/artifacts/fedora/fedora.go index 36992bc0..9ecedd9f 100644 --- a/artifacts/fedora/fedora.go +++ b/artifacts/fedora/fedora.go @@ -77,6 +77,7 @@ func (f *fedora) Inspect() (*api.ArtifactDetails, error) { SHA256Sum: release.Sha256, DownloadURL: release.Link, AdditionalUniqueTags: []string{additionalTag}, + ImageArchitecture: "amd64", }, nil } } @@ -105,7 +106,7 @@ func (f *fedora) Tests() []api.ArtifactTest { } } -func (f *fedoraGatherer) Gather() ([]api.Artifact, error) { +func (f *fedoraGatherer) Gather() ([][]api.Artifact, error) { releases, err := getReleases(f.getter) if err != nil { return nil, fmt.Errorf("error getting releases: %v", err) @@ -126,7 +127,7 @@ func (f *fedoraGatherer) Gather() ([]api.Artifact, error) { } } - return artifacts, nil + return [][]api.Artifact{artifacts}, nil } func getReleases(getter http.Getter) (Releases, error) { diff --git a/artifacts/fedora/fedora_test.go b/artifacts/fedora/fedora_test.go index 6a55d352..de7f040d 100644 --- a/artifacts/fedora/fedora_test.go +++ b/artifacts/fedora/fedora_test.go @@ -28,6 +28,7 @@ var _ = Describe("Fedora", func() { SHA256Sum: "fe84502779b3477284a8d4c86731f642ca10dd3984d2b5eccdf82630a9ca2de6", DownloadURL: "https://download.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2", //nolint:lll AdditionalUniqueTags: []string{"35-1.2"}, + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", @@ -51,6 +52,7 @@ var _ = Describe("Fedora", func() { SHA256Sum: "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", DownloadURL: "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", //nolint:lll AdditionalUniqueTags: []string{"34-1.2"}, + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", @@ -72,7 +74,7 @@ var _ = Describe("Fedora", func() { ) It("Gather should be able to parse releases files", func() { - artifacts := []api.Artifact{ + artifacts := [][]api.Artifact{{ &fedora{ Version: "36", Arch: "x86_64", @@ -93,7 +95,7 @@ var _ = Describe("Fedora", func() { common.DefaultPreferenceEnv: "fedora", }, }, - } + }} c := NewGatherer() c.getter = testutil.NewMockGetter("testdata/releases.json") diff --git a/artifacts/ubuntu/ubuntu.go b/artifacts/ubuntu/ubuntu.go index 67c01bd3..6c51f02a 100644 --- a/artifacts/ubuntu/ubuntu.go +++ b/artifacts/ubuntu/ubuntu.go @@ -52,9 +52,10 @@ func (u *ubuntu) Inspect() (*api.ArtifactDetails, error) { } if checksum, exists := checksums[u.Variant]; exists { return &api.ArtifactDetails{ - SHA256Sum: checksum, - DownloadURL: baseURL + u.Variant, - Compression: u.Compression, + SHA256Sum: checksum, + DownloadURL: baseURL + u.Variant, + Compression: u.Compression, + ImageArchitecture: "amd64", }, nil } return nil, fmt.Errorf("file %q does not exist in the SHA256SUMS file: %v", u.Variant, err) diff --git a/artifacts/ubuntu/ubuntu_test.go b/artifacts/ubuntu/ubuntu_test.go index cd97be07..24f64539 100644 --- a/artifacts/ubuntu/ubuntu_test.go +++ b/artifacts/ubuntu/ubuntu_test.go @@ -24,8 +24,9 @@ var _ = Describe("Ubuntu", func() { }, Entry("ubuntu:22.04", "22.04", "testdata/SHA256SUM", &api.ArtifactDetails{ - SHA256Sum: "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd", - DownloadURL: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img", + SHA256Sum: "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd", + DownloadURL: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img", + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", diff --git a/cmd/medius/common/registry.go b/cmd/medius/common/registry.go index 101a77cf..4090960b 100644 --- a/cmd/medius/common/registry.go +++ b/cmd/medius/common/registry.go @@ -16,7 +16,7 @@ import ( ) type Entry struct { - Artifact api.Artifact + Artifacts []api.Artifact UseForDocs bool UseForLatest bool SkipWhenNotFocused bool @@ -24,69 +24,73 @@ type Entry struct { var staticRegistry = []Entry{ { - Artifact: centos.New("8.4", nil), + Artifacts: []api.Artifact{ + centos.New("8.4", nil), + }, UseForDocs: false, }, { - Artifact: centos.New( - "7-2009", - defaultEnvVariables("u1.small", "centos.7"), - ), + Artifacts: []api.Artifact{ + centos.New("7-2009", defaultEnvVariables("u1.small", "centos.7")), + }, UseForDocs: true, }, { - Artifact: centosstream.New( - "9", - &docs.UserData{ - Username: "cloud-user", - }, - defaultEnvVariables("u1.small", "centos.stream9"), - ), + Artifacts: []api.Artifact{ + centosstream.New("9", &docs.UserData{Username: "cloud-user"}, defaultEnvVariables("u1.small", "centos.stream9")), + }, UseForDocs: true, }, { - Artifact: centosstream.New( - "8", - &docs.UserData{ - Username: "centos", - }, - defaultEnvVariables("u1.small", "centos.stream8"), - ), + Artifacts: []api.Artifact{ + centosstream.New("8", &docs.UserData{Username: "centos"}, defaultEnvVariables("u1.small", "centos.stream8")), + }, UseForDocs: false, }, { - Artifact: ubuntu.New( - "22.04", - defaultEnvVariables("u1.small", "ubuntu"), - ), + Artifacts: []api.Artifact{ + ubuntu.New("22.04", defaultEnvVariables("u1.small", "ubuntu")), + }, UseForDocs: true, }, { - Artifact: ubuntu.New( - "20.04", - defaultEnvVariables("u1.small", "ubuntu"), - ), + Artifacts: []api.Artifact{ + ubuntu.New("20.04", defaultEnvVariables("u1.small", "ubuntu")), + }, UseForDocs: false, }, { - Artifact: ubuntu.New( - "18.04", - defaultEnvVariables("u1.small", "ubuntu"), - ), + Artifacts: []api.Artifact{ + ubuntu.New("18.04", defaultEnvVariables("u1.small", "ubuntu")), + }, UseForDocs: false, }, // for testing only { - Artifact: generic.New( - &api.ArtifactDetails{ - SHA256Sum: "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1", - DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img", - }, - &api.Metadata{ - Name: "cirros", - Version: "6.1", - }, - ), + Artifacts: []api.Artifact{ + generic.New( + &api.ArtifactDetails{ + SHA256Sum: "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1", + DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img", + ImageArchitecture: "amd64", + }, + &api.Metadata{ + Name: "cirros", + Version: "6.1", + }, + ), + generic.New( + &api.ArtifactDetails{ + SHA256Sum: "db9420c481c11dee17860aa46fb1a3efa05fa4fb152726d6344e24da03cb0ccf", + DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-aarch64-disk.img", + ImageArchitecture: "arm64", + }, + &api.Metadata{ + Name: "cirros", + Version: "6.1", + }, + ), + }, SkipWhenNotFocused: true, UseForDocs: false, }, @@ -100,7 +104,7 @@ func gatherArtifacts(registry *[]Entry, gatherers []api.ArtifactsGatherer) { } else { for i := range artifacts { *registry = append(*registry, Entry{ - Artifact: artifacts[i], + Artifacts: []api.Artifact{artifacts[0][i]}, UseForDocs: i == 0, UseForLatest: i == 0, }) @@ -131,12 +135,16 @@ func ShouldSkip(focus string, entry *Entry) bool { return entry.SkipWhenNotFocused } + if len(entry.Artifacts) == 0 { + return true + } + focusSplit := strings.Split(focus, ":") wildcardFocus := len(focusSplit) == 2 && focusSplit[1] == "*" if wildcardFocus { - return focusSplit[0] != entry.Artifact.Metadata().Name + return focusSplit[0] != entry.Artifacts[0].Metadata().Name } - return focus != entry.Artifact.Metadata().Describe() + return focus != entry.Artifacts[0].Metadata().Describe() } diff --git a/cmd/medius/docs/docs.go b/cmd/medius/docs/docs.go index 0c41d63d..62b6e3e3 100644 --- a/cmd/medius/docs/docs.go +++ b/cmd/medius/docs/docs.go @@ -60,10 +60,12 @@ func run(options *common.Options) error { } focusMatched = true - log := common.Logger(p.Artifact) - name := p.Artifact.Metadata().Name - description, err := createDescription(p.Artifact, options.PublishDocsOptions.Registry) + artifact := p.Artifacts[0] + log := common.Logger(artifact) + name := artifact.Metadata().Name + + description, err := createDescription(artifact, options.PublishDocsOptions.Registry) if err != nil { success = false log.Errorf("error marshaling example for %q: %v", name, err) diff --git a/cmd/medius/images/common.go b/cmd/medius/images/common.go index 41b5844f..c137d57c 100644 --- a/cmd/medius/images/common.go +++ b/cmd/medius/images/common.go @@ -44,15 +44,16 @@ func spawnWorkers(ctx context.Context, o *common.Options, go func() { defer wg.Done() for e := range jobChan { + artifact := e.Artifacts[0] result, workerErr := fn(e) if result != nil { resultsChan <- workerResult{ - Key: e.Artifact.Metadata().Describe(), + Key: artifact.Metadata().Describe(), Value: *result, } } if workerErr != nil && !errors.Is(workerErr, context.Canceled) { - common.Logger(e.Artifact).Error(workerErr) + common.Logger(artifact).Error(workerErr) errChan <- workerErr } if errors.Is(ctx.Err(), context.Canceled) { diff --git a/cmd/medius/images/promote.go b/cmd/medius/images/promote.go index 91848a17..06d85883 100644 --- a/cmd/medius/images/promote.go +++ b/cmd/medius/images/promote.go @@ -29,7 +29,8 @@ func NewPromoteImagesCommand(options *common.Options) *cobra.Command { } focusMatched, resultsChan, workerErr := spawnWorkers(cmd.Context(), options, func(e *common.Entry) (*api.ArtifactResult, error) { - description := e.Artifact.Metadata().Describe() + artifact := e.Artifacts[0] + description := artifact.Metadata().Describe() r, ok := results[description] if !ok { return nil, nil @@ -42,7 +43,7 @@ func NewPromoteImagesCommand(options *common.Options) *cobra.Command { } errString := "" - err := promoteArtifact(cmd.Context(), e.Artifact, r.Tags, options) + err := promoteArtifact(cmd.Context(), artifact, r.Tags, options) if err != nil { errString = err.Error() } diff --git a/cmd/medius/images/push.go b/cmd/medius/images/push.go index 00939719..36455d34 100644 --- a/cmd/medius/images/push.go +++ b/cmd/medius/images/push.go @@ -46,10 +46,11 @@ func NewPublishImagesCommand(options *common.Options) *cobra.Command { focusMatched, resultsChan, workerErr := spawnWorkers(cmd.Context(), options, func(e *common.Entry) (*api.ArtifactResult, error) { errString := "" + artifact := e.Artifacts[0] b := buildAndPublish{ Ctx: cmd.Context(), - Log: common.Logger(e.Artifact), + Log: common.Logger(artifact), Options: options, Repo: &repository.RepositoryImpl{}, Getter: &http.HTTPGetter{}, @@ -107,18 +108,17 @@ func NewPublishImagesCommand(options *common.Options) *cobra.Command { } func (b *buildAndPublish) Do(entry *common.Entry, timestamp time.Time) ([]string, error) { - metadata := entry.Artifact.Metadata() - artifactInfo, err := entry.Artifact.Inspect() + metadata := entry.Artifacts[0].Metadata() + artifactInfo, err := entry.Artifacts[0].Inspect() if err != nil { return nil, fmt.Errorf("error introspecting artifact %q: %v", metadata.Describe(), err) } - b.Log.Infof("Remote artifact checksum: %q", artifactInfo.SHA256Sum) - imageSha, err := b.getImageSha(metadata.Describe()) + rebuildNeeded, err := b.rebuildNeeded(entry) if err != nil { return nil, err } - if imageSha == artifactInfo.SHA256Sum && !b.Options.PublishImagesOptions.ForceBuild { + if !rebuildNeeded && !b.Options.PublishImagesOptions.ForceBuild { b.Log.Info("Nothing to do.") return nil, nil } @@ -126,26 +126,26 @@ func (b *buildAndPublish) Do(entry *common.Entry, timestamp time.Time) ([]string return nil, b.Ctx.Err() } - b.Log.Infof("Rebuild needed, downloading %q ...", artifactInfo.DownloadURL) - file, err := b.getArtifact(artifactInfo) + images, artifacts, err := b.buildImages(entry) if err != nil { return nil, err } - defer os.Remove(file) - - b.Log.Info("Building containerdisk ...") - containerDisk, err := build.ContainerDisk(file, build.ContainerDiskConfig(artifactInfo.SHA256Sum, metadata.EnvVariables)) - if err != nil { - return nil, fmt.Errorf("error creating the containerdisk : %v", err) - } - if errors.Is(b.Ctx.Err(), context.Canceled) { - return nil, b.Ctx.Err() - } + defer cleanupArtifacts(artifacts) names := prepareTags(timestamp, b.Options.PublishImagesOptions.TargetRegistry, entry, artifactInfo) for _, name := range names { - if err := b.pushImage(containerDisk, name); err != nil { - return nil, err + if len(images) > 1 { + containerDiskIndex, err := build.ContainerDiskIndex(images) + if err != nil { + return nil, fmt.Errorf("error creating the containerdisk index : %v", err) + } + if err := b.pushImageIndex(containerDiskIndex, name); err != nil { + return nil, err + } + } else if len(images) == 1 { + if err := b.pushImage(images[0], name); err != nil { + return nil, err + } } if errors.Is(b.Ctx.Err(), context.Canceled) { return nil, b.Ctx.Err() @@ -155,9 +155,9 @@ func (b *buildAndPublish) Do(entry *common.Entry, timestamp time.Time) ([]string return prepareTags(timestamp, "", entry, artifactInfo), nil } -func (b *buildAndPublish) getImageSha(description string) (imageSha string, err error) { +func (b *buildAndPublish) getImageSha(description, arch string) (imageSha string, err error) { imageName := path.Join(b.Options.PublishImagesOptions.SourceRegistry, description) - imageInfo, err := b.Repo.ImageMetadata(imageName, b.Options.AllowInsecureRegistry) + imageInfo, err := b.Repo.ImageMetadata(imageName, arch, b.Options.AllowInsecureRegistry) if err != nil { err = b.handleMetadataError(imageName, err) } else { @@ -247,6 +247,65 @@ func (b *buildAndPublish) readArtifact(artifactReader http.ReadCloserWithChecksu return file.Name(), nil } +func (b *buildAndPublish) buildImages(entry *common.Entry) ([]v1.Image, []string, error) { + var images []v1.Image + var artifacts []string + + for i := range entry.Artifacts { + metadata := entry.Artifacts[i].Metadata() + artifactInfo, err := entry.Artifacts[i].Inspect() + if err != nil { + return nil, nil, fmt.Errorf("error introspecting artifact %q: %v", metadata.Describe(), err) + } + + b.Log.Infof("Rebuild needed, downloading %q ...", artifactInfo.DownloadURL) + file, err := b.getArtifact(artifactInfo) + if err != nil { + return nil, nil, err + } + artifacts = append(artifacts, file) + + b.Log.Info("Building containerdisk ...") + image, err := build.ContainerDisk(file, + artifactInfo.ImageArchitecture, + build.ContainerDiskConfig(artifactInfo.SHA256Sum, metadata.EnvVariables)) + if err != nil { + return nil, nil, fmt.Errorf("error creating the containerdisk : %v", err) + } + if errors.Is(b.Ctx.Err(), context.Canceled) { + return nil, nil, b.Ctx.Err() + } + images = append(images, image) + } + + return images, artifacts, nil +} + +func (b *buildAndPublish) rebuildNeeded(entry *common.Entry) (bool, error) { + if len(entry.Artifacts) == 0 { + err := errors.New("entry has no artifacts to check for rebuild") + b.Log.Error(err) + return false, err + } + + for i := range entry.Artifacts { + metadata := entry.Artifacts[i].Metadata() + artifactInfo, err := entry.Artifacts[i].Inspect() + if err != nil { + return false, fmt.Errorf("error introspecting artifact %q: %v", metadata.Describe(), err) + } + imageSha, err := b.getImageSha(metadata.Describe(), artifactInfo.ImageArchitecture) + if err != nil { + return false, err + } + if imageSha != artifactInfo.SHA256Sum { + return true, nil + } + } + + return false, nil +} + func (b *buildAndPublish) pushImage(containerDisk v1.Image, name string) error { if !b.Options.DryRun { b.Log.Infof("Pushing %s", name) @@ -261,8 +320,22 @@ func (b *buildAndPublish) pushImage(containerDisk v1.Image, name string) error { return nil } +func (b *buildAndPublish) pushImageIndex(containerDiskIndex v1.ImageIndex, name string) error { + if !b.Options.DryRun { + b.Log.Infof("Pushing %s image index", name) + if err := b.Repo.PushImageIndex(b.Ctx, containerDiskIndex, name); err != nil { + b.Log.WithError(err).Error("Failed to push image image") + return err + } + } else { + b.Log.Infof("Dry run enabled, not pushing %s image index", name) + } + + return nil +} + func prepareTags(timestamp time.Time, registry string, entry *common.Entry, artifactDetails *api.ArtifactDetails) []string { - metadata := entry.Artifact.Metadata() + metadata := entry.Artifacts[0].Metadata() imageName := path.Join(registry, metadata.Describe()) names := []string{fmt.Sprintf("%s-%s", imageName, timestamp.Format("0601021504"))} @@ -281,3 +354,9 @@ func prepareTags(timestamp time.Time, registry string, entry *common.Entry, arti return names } + +func cleanupArtifacts(artifacts []string) { + for _, file := range artifacts { + os.Remove(file) + } +} diff --git a/cmd/medius/images/verify.go b/cmd/medius/images/verify.go index 58df0570..2ed0499e 100644 --- a/cmd/medius/images/verify.go +++ b/cmd/medius/images/verify.go @@ -48,7 +48,8 @@ func NewVerifyImagesCommand(options *common.Options) *cobra.Command { } focusMatched, resultsChan, workerErr := spawnWorkers(cmd.Context(), options, func(e *common.Entry) (*api.ArtifactResult, error) { - description := e.Artifact.Metadata().Describe() + artifact := e.Artifacts[0] + description := artifact.Metadata().Describe() r, ok := results[description] if !ok { return nil, nil @@ -61,7 +62,7 @@ func NewVerifyImagesCommand(options *common.Options) *cobra.Command { } errString := "" - err := verifyArtifact(cmd.Context(), e.Artifact, r, options, client) + err := verifyArtifact(cmd.Context(), artifact, r, options, client) if err != nil { errString = err.Error() } diff --git a/pkg/api/artifact.go b/pkg/api/artifact.go index 961ec1ba..fb124ae0 100644 --- a/pkg/api/artifact.go +++ b/pkg/api/artifact.go @@ -32,6 +32,8 @@ type ArtifactDetails struct { SHA256Sum string // DownloadURL points to the target image. DownloadURL string + // ImageArchitecture is the target architecture of the image. + ImageArchitecture string // Compression describes the compression format of the downloaded image. // Supported are "" (none), "gzip" and "xz". Compression string @@ -70,5 +72,5 @@ type Artifact interface { type ArtifactsGatherer interface { // Gather must return a sorted list of dynamically gathered artifacts. // Artifacts have to be sorted in descending order with the latest release coming first. - Gather() ([]Artifact, error) + Gather() ([][]Artifact, error) } diff --git a/pkg/build/build.go b/pkg/build/build.go index ce93299c..4f221194 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -6,12 +6,13 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/tarball" ) const ( - LabelShaSum = "shasum" - ImageArchitecture = "amd64" + LabelShaSum = "shasum" + ImageOS = "linux" ) func ContainerDiskConfig(checksum string, envVariables map[string]string) v1.Config { @@ -27,7 +28,7 @@ func ContainerDiskConfig(checksum string, envVariables map[string]string) v1.Con return v1.Config{Labels: labels, Env: env} } -func ContainerDisk(imgPath string, config v1.Config) (v1.Image, error) { +func ContainerDisk(imgPath, imgArch string, config v1.Config) (v1.Image, error) { img := empty.Image layer, err := tarball.LayerFromOpener(StreamLayerOpener(imgPath)) if err != nil { @@ -45,7 +46,8 @@ func ContainerDisk(imgPath string, config v1.Config) (v1.Image, error) { } // Modify the config file - cf.Architecture = ImageArchitecture + cf.Architecture = imgArch + cf.OS = ImageOS cf.Config = config img, err = mutate.ConfigFile(img, cf) @@ -55,3 +57,27 @@ func ContainerDisk(imgPath string, config v1.Config) (v1.Image, error) { return img, nil } + +func ContainerDiskIndex(images []v1.Image) (v1.ImageIndex, error) { + var indexAddendum []mutate.IndexAddendum + + for _, image := range images { + configFile, err := image.ConfigFile() + if err != nil { + return nil, err + } + + descriptor, err := partial.Descriptor(image) + if err != nil { + return nil, err + } + descriptor.Platform = configFile.Platform() + + indexAddendum = append(indexAddendum, mutate.IndexAddendum{ + Add: image, + Descriptor: *descriptor, + }) + } + + return mutate.AppendManifests(empty.Index, indexAddendum...), nil +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index d1b2c63f..fc62378e 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -12,7 +12,9 @@ import ( "github.com/docker/distribution/registry/api/errcode" v2 "github.com/docker/distribution/registry/api/v2" "github.com/google/go-containerregistry/pkg/crane" + crname "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/pkg/errors" ) @@ -28,17 +30,20 @@ type ImageInfo struct { } type Repository interface { - ImageMetadata(imgRef string, insecure bool) (*ImageInfo, error) + ImageMetadata(imgRef, arch string, insecure bool) (*ImageInfo, error) PushImage(ctx context.Context, img v1.Image, imgRef string) error + PushImageIndex(ctx context.Context, img v1.ImageIndex, imgRef string) error CopyImage(ctx context.Context, srcRef, dstRef string, insecure bool) error } type RepositoryImpl struct { } -func (r RepositoryImpl) ImageMetadata(imgRef string, insecure bool) (imageInfo *ImageInfo, retErr error) { +func (r RepositoryImpl) ImageMetadata(imgRef, arch string, insecure bool) (imageInfo *ImageInfo, retErr error) { sys := &types.SystemContext{ OCIInsecureSkipTLSVerify: insecure, + ArchitectureChoice: arch, + OSChoice: "linux", } if insecure { sys.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue @@ -82,6 +87,15 @@ func (r RepositoryImpl) PushImage(ctx context.Context, img v1.Image, imgRef stri return crane.Push(img, imgRef, crane.WithContext(ctx)) } +func (r RepositoryImpl) PushImageIndex(ctx context.Context, imageIndex v1.ImageIndex, imageRef string) error { + ref, err := crname.ParseReference(imageRef) + if err != nil { + return err + } + + return remote.WriteIndex(ref, imageIndex, crane.GetOptions().Remote...) +} + func (r RepositoryImpl) CopyImage(ctx context.Context, srcRef, dstRef string, insecure bool) error { options := []crane.Option{ crane.WithContext(ctx),