feat: build and push image index

Push an image index to have multiple image manifests, instead of single image manifest when needed. Jira-Url: https://issues.redhat.com/browse/CNV-38597 Signed-off-by: Ben Oukhanov <[email protected]>
kubevirt · Mar 27, 2024 · 9e8fecf · 9e8fecf
1 parent 8745da2
commit 9e8fecf
Show file tree

Hide file tree

Showing 17 changed files with 244 additions and 97 deletions.
diff --git a/artifacts/centos/centos.go b/artifacts/centos/centos.go
@@ -64,6 +64,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) {
 			SHA256Sum:            checksum,
 			DownloadURL:          baseURL + candidate,
 			AdditionalUniqueTags: getAdditionalTags(c.Version, c.Variant, candidate),
+			ImageArchitecture:    "amd64",
 		}, nil
 	}
 

diff --git a/artifacts/centos/centos_test.go b/artifacts/centos/centos_test.go
@@ -28,6 +28,7 @@ var _ = Describe("Centos", func() {
 				DownloadURL:          "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2",
 				Compression:          "",
 				AdditionalUniqueTags: []string{"8.4.2105-20210603.0", "8.4.2105"},
+				ImageArchitecture:    "amd64",
 			},
 			map[string]string{
 				"TEST_ENV_VAR": "test-value",
@@ -50,6 +51,7 @@ var _ = Describe("Centos", func() {
 				DownloadURL:          "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2",
 				Compression:          "",
 				AdditionalUniqueTags: []string{"8.3.2011-20201204.2", "8.3.2011"},
+				ImageArchitecture:    "amd64",
 			},
 			map[string]string{
 				"TEST_ENV_VAR": "test-value",
@@ -68,8 +70,9 @@ var _ = Describe("Centos", func() {
 		),
 		Entry("centos:7-2009", "7-2009", "testdata/centos7.checksum",
 			&api.ArtifactDetails{
-				SHA256Sum:   "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3",
-				DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2",
+				SHA256Sum:         "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3",
+				DownloadURL:       "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2",
+				ImageArchitecture: "amd64",
 			},
 			map[string]string{
 				common.DefaultInstancetypeEnv: "u1.small",
@@ -90,8 +93,9 @@ var _ = Describe("Centos", func() {
 		),
 		Entry("centos:7-1809", "7-1809", "testdata/centos7.checksum",
 			&api.ArtifactDetails{
-				SHA256Sum:   "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2",
-				DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2",
+				SHA256Sum:         "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2",
+				DownloadURL:       "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2",
+				ImageArchitecture: "amd64",
 			},
 			map[string]string{
 				common.DefaultInstancetypeEnv: "u1.small",

diff --git a/artifacts/centosstream/centos-stream.go b/artifacts/centosstream/centos-stream.go
@@ -89,6 +89,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) {
 			SHA256Sum:            checksum,
 			DownloadURL:          baseURL + candidate,
 			AdditionalUniqueTags: additionalTags,
+			ImageArchitecture:    "amd64",
 		}, nil
 	}
 

diff --git a/artifacts/centosstream/centos-stream_test.go b/artifacts/centosstream/centos-stream_test.go
@@ -28,6 +28,7 @@ var _ = Describe("CentosStream", func() {
 				SHA256Sum:            "8e22e67687b81e38c7212fc30c47cb24cbc4935c0f2459ed139f498397d1e7cd",
 				DownloadURL:          "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210603.0.x86_64.qcow2",
 				AdditionalUniqueTags: []string{"8-20210603.0"},
+				ImageArchitecture:    "amd64",
 			},
 			&docs.UserData{
 				Username: "centos",
@@ -54,6 +55,7 @@ var _ = Describe("CentosStream", func() {
 				SHA256Sum:            "bcebdc00511d6e18782732570056cfbc7cba318302748bfc8f66be9c0db68142",
 				DownloadURL:          "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20211222.0.x86_64.qcow2",
 				AdditionalUniqueTags: []string{"9-20211222.0"},
+				ImageArchitecture:    "amd64",
 			},
 			&docs.UserData{
 				Username: "cloud-user",

diff --git a/artifacts/fedora/fedora.go b/artifacts/fedora/fedora.go
@@ -77,6 +77,7 @@ func (f *fedora) Inspect() (*api.ArtifactDetails, error) {
 				SHA256Sum:            release.Sha256,
 				DownloadURL:          release.Link,
 				AdditionalUniqueTags: []string{additionalTag},
+				ImageArchitecture:    "amd64",
 			}, nil
 		}
 	}
@@ -105,7 +106,7 @@ func (f *fedora) Tests() []api.ArtifactTest {
 	}
 }
 
-func (f *fedoraGatherer) Gather() ([]api.Artifact, error) {
+func (f *fedoraGatherer) Gather() ([][]api.Artifact, error) {
 	releases, err := getReleases(f.getter)
 	if err != nil {
 		return nil, fmt.Errorf("error getting releases: %v", err)
@@ -126,7 +127,7 @@ func (f *fedoraGatherer) Gather() ([]api.Artifact, error) {
 		}
 	}
 
-	return artifacts, nil
+	return [][]api.Artifact{artifacts}, nil
 }
 
 func getReleases(getter http.Getter) (Releases, error) {

diff --git a/artifacts/fedora/fedora_test.go b/artifacts/fedora/fedora_test.go
@@ -28,6 +28,7 @@ var _ = Describe("Fedora", func() {
 				SHA256Sum:            "fe84502779b3477284a8d4c86731f642ca10dd3984d2b5eccdf82630a9ca2de6",
 				DownloadURL:          "https://download.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2", //nolint:lll
 				AdditionalUniqueTags: []string{"35-1.2"},
+				ImageArchitecture:    "amd64",
 			},
 			map[string]string{
 				common.DefaultInstancetypeEnv: "u1.small",
@@ -51,6 +52,7 @@ var _ = Describe("Fedora", func() {
 				SHA256Sum:            "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea",
 				DownloadURL:          "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", //nolint:lll
 				AdditionalUniqueTags: []string{"34-1.2"},
+				ImageArchitecture:    "amd64",
 			},
 			map[string]string{
 				common.DefaultInstancetypeEnv: "u1.small",
@@ -72,7 +74,7 @@ var _ = Describe("Fedora", func() {
 	)
 
 	It("Gather should be able to parse releases files", func() {
-		artifacts := []api.Artifact{
+		artifacts := [][]api.Artifact{{
 			&fedora{
 				Version: "36",
 				Arch:    "x86_64",
@@ -93,7 +95,7 @@ var _ = Describe("Fedora", func() {
 					common.DefaultPreferenceEnv:   "fedora",
 				},
 			},
-		}
+		}}
 
 		c := NewGatherer()
 		c.getter = testutil.NewMockGetter("testdata/releases.json")

diff --git a/artifacts/ubuntu/ubuntu.go b/artifacts/ubuntu/ubuntu.go
@@ -52,9 +52,10 @@ func (u *ubuntu) Inspect() (*api.ArtifactDetails, error) {
 	}
 	if checksum, exists := checksums[u.Variant]; exists {
 		return &api.ArtifactDetails{
-			SHA256Sum:   checksum,
-			DownloadURL: baseURL + u.Variant,
-			Compression: u.Compression,
+			SHA256Sum:         checksum,
+			DownloadURL:       baseURL + u.Variant,
+			Compression:       u.Compression,
+			ImageArchitecture: "amd64",
 		}, nil
 	}
 	return nil, fmt.Errorf("file %q does not exist in the SHA256SUMS file: %v", u.Variant, err)

diff --git a/artifacts/ubuntu/ubuntu_test.go b/artifacts/ubuntu/ubuntu_test.go
@@ -24,8 +24,9 @@ var _ = Describe("Ubuntu", func() {
 		},
 		Entry("ubuntu:22.04", "22.04", "testdata/SHA256SUM",
 			&api.ArtifactDetails{
-				SHA256Sum:   "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd",
-				DownloadURL: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img",
+				SHA256Sum:         "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd",
+				DownloadURL:       "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img",
+				ImageArchitecture: "amd64",
 			},
 			map[string]string{
 				common.DefaultInstancetypeEnv: "u1.small",

diff --git a/cmd/medius/common/registry.go b/cmd/medius/common/registry.go
@@ -16,77 +16,81 @@ import (
 )
 
 type Entry struct {
-	Artifact           api.Artifact
+	Artifacts          []api.Artifact
 	UseForDocs         bool
 	UseForLatest       bool
 	SkipWhenNotFocused bool
 }
 
 var staticRegistry = []Entry{
 	{
-		Artifact:   centos.New("8.4", nil),
+		Artifacts: []api.Artifact{
+			centos.New("8.4", nil),
+		},
 		UseForDocs: false,
 	},
 	{
-		Artifact: centos.New(
-			"7-2009",
-			defaultEnvVariables("u1.small", "centos.7"),
-		),
+		Artifacts: []api.Artifact{
+			centos.New("7-2009", defaultEnvVariables("u1.small", "centos.7")),
+		},
 		UseForDocs: true,
 	},
 	{
-		Artifact: centosstream.New(
-			"9",
-			&docs.UserData{
-				Username: "cloud-user",
-			},
-			defaultEnvVariables("u1.small", "centos.stream9"),
-		),
+		Artifacts: []api.Artifact{
+			centosstream.New("9", &docs.UserData{Username: "cloud-user"}, defaultEnvVariables("u1.small", "centos.stream9")),
+		},
 		UseForDocs: true,
 	},
 	{
-		Artifact: centosstream.New(
-			"8",
-			&docs.UserData{
-				Username: "centos",
-			},
-			defaultEnvVariables("u1.small", "centos.stream8"),
-		),
+		Artifacts: []api.Artifact{
+			centosstream.New("8", &docs.UserData{Username: "centos"}, defaultEnvVariables("u1.small", "centos.stream8")),
+		},
 		UseForDocs: false,
 	},
 	{
-		Artifact: ubuntu.New(
-			"22.04",
-			defaultEnvVariables("u1.small", "ubuntu"),
-		),
+		Artifacts: []api.Artifact{
+			ubuntu.New("22.04", defaultEnvVariables("u1.small", "ubuntu")),
+		},
 		UseForDocs: true,
 	},
 	{
-		Artifact: ubuntu.New(
-			"20.04",
-			defaultEnvVariables("u1.small", "ubuntu"),
-		),
+		Artifacts: []api.Artifact{
+			ubuntu.New("20.04", defaultEnvVariables("u1.small", "ubuntu")),
+		},
 		UseForDocs: false,
 	},
 	{
-		Artifact: ubuntu.New(
-			"18.04",
-			defaultEnvVariables("u1.small", "ubuntu"),
-		),
+		Artifacts: []api.Artifact{
+			ubuntu.New("18.04", defaultEnvVariables("u1.small", "ubuntu")),
+		},
 		UseForDocs: false,
 	},
 	// for testing only
 	{
-		Artifact: generic.New(
-			&api.ArtifactDetails{
-				SHA256Sum:   "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1",
-				DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img",
-			},
-			&api.Metadata{
-				Name:    "cirros",
-				Version: "6.1",
-			},
-		),
+		Artifacts: []api.Artifact{
+			generic.New(
+				&api.ArtifactDetails{
+					SHA256Sum:         "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1",
+					DownloadURL:       "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img",
+					ImageArchitecture: "amd64",
+				},
+				&api.Metadata{
+					Name:    "cirros",
+					Version: "6.1",
+				},
+			),
+			generic.New(
+				&api.ArtifactDetails{
+					SHA256Sum:         "db9420c481c11dee17860aa46fb1a3efa05fa4fb152726d6344e24da03cb0ccf",
+					DownloadURL:       "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-aarch64-disk.img",
+					ImageArchitecture: "arm64",
+				},
+				&api.Metadata{
+					Name:    "cirros",
+					Version: "6.1",
+				},
+			),
+		},
 		SkipWhenNotFocused: true,
 		UseForDocs:         false,
 	},
@@ -100,7 +104,7 @@ func gatherArtifacts(registry *[]Entry, gatherers []api.ArtifactsGatherer) {
 		} else {
 			for i := range artifacts {
 				*registry = append(*registry, Entry{
-					Artifact:     artifacts[i],
+					Artifacts:    []api.Artifact{artifacts[0][i]},
 					UseForDocs:   i == 0,
 					UseForLatest: i == 0,
 				})
@@ -131,12 +135,16 @@ func ShouldSkip(focus string, entry *Entry) bool {
 		return entry.SkipWhenNotFocused
 	}
 
+	if len(entry.Artifacts) == 0 {
+		return true
+	}
+
 	focusSplit := strings.Split(focus, ":")
 	wildcardFocus := len(focusSplit) == 2 && focusSplit[1] == "*"
 
 	if wildcardFocus {
-		return focusSplit[0] != entry.Artifact.Metadata().Name
+		return focusSplit[0] != entry.Artifacts[0].Metadata().Name
 	}
 
-	return focus != entry.Artifact.Metadata().Describe()
+	return focus != entry.Artifacts[0].Metadata().Describe()
 }
diff --git a/cmd/medius/docs/docs.go b/cmd/medius/docs/docs.go
@@ -60,10 +60,12 @@ func run(options *common.Options) error {
 		}
 
 		focusMatched = true
-		log := common.Logger(p.Artifact)
-		name := p.Artifact.Metadata().Name
 
-		description, err := createDescription(p.Artifact, options.PublishDocsOptions.Registry)
+		artifact := p.Artifacts[0]
+		log := common.Logger(artifact)
+		name := artifact.Metadata().Name
+
+		description, err := createDescription(artifact, options.PublishDocsOptions.Registry)
 		if err != nil {
 			success = false
 			log.Errorf("error marshaling example for %q: %v", name, err)

diff --git a/cmd/medius/images/common.go b/cmd/medius/images/common.go
@@ -44,15 +44,16 @@ func spawnWorkers(ctx context.Context, o *common.Options,
 		go func() {
 			defer wg.Done()
 			for e := range jobChan {
+				artifact := e.Artifacts[0]
 				result, workerErr := fn(e)
 				if result != nil {
 					resultsChan <- workerResult{
-						Key:   e.Artifact.Metadata().Describe(),
+						Key:   artifact.Metadata().Describe(),
 						Value: *result,
 					}
 				}
 				if workerErr != nil && !errors.Is(workerErr, context.Canceled) {
-					common.Logger(e.Artifact).Error(workerErr)
+					common.Logger(artifact).Error(workerErr)
 					errChan <- workerErr
 				}
 				if errors.Is(ctx.Err(), context.Canceled) {

diff --git a/cmd/medius/images/promote.go b/cmd/medius/images/promote.go
@@ -29,7 +29,8 @@ func NewPromoteImagesCommand(options *common.Options) *cobra.Command {
 			}
 
 			focusMatched, resultsChan, workerErr := spawnWorkers(cmd.Context(), options, func(e *common.Entry) (*api.ArtifactResult, error) {
-				description := e.Artifact.Metadata().Describe()
+				artifact := e.Artifacts[0]
+				description := artifact.Metadata().Describe()
 				r, ok := results[description]
 				if !ok {
 					return nil, nil
@@ -42,7 +43,7 @@ func NewPromoteImagesCommand(options *common.Options) *cobra.Command {
 				}
 
 				errString := ""
-				err := promoteArtifact(cmd.Context(), e.Artifact, r.Tags, options)
+				err := promoteArtifact(cmd.Context(), artifact, r.Tags, options)
 				if err != nil {
 					errString = err.Error()
 				}