From 925849768eb2d94aebe785062b9b2116257fe8f3 Mon Sep 17 00:00:00 2001 From: Marc Khouzam Date: Wed, 2 Aug 2023 07:04:48 -0400 Subject: [PATCH] Build plugins for ARM64 (#367) We start building the 'builder' and 'test' plugins for ARM64. We also default new plugin projects to also building for ARM64. The builder plugin will publish the ARM64 binaries and include them in the plugin inventory DB if they are present. If they are not present, the builder plugin will ignore them with a printout. Signed-off-by: Marc Khouzam --- cmd/plugin/builder/helpers/helper.go | 11 +++++ .../builder/inventory/inventory_plugin.go | 40 ++++++++++++++----- .../plugintemplates/plugin-tooling.mk.tmpl | 2 +- plugin-tooling.mk | 2 +- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/cmd/plugin/builder/helpers/helper.go b/cmd/plugin/builder/helpers/helper.go index 67544dd0a..568d29699 100644 --- a/cmd/plugin/builder/helpers/helper.go +++ b/cmd/plugin/builder/helpers/helper.go @@ -97,3 +97,14 @@ func GetNumberOfIndividualPluginBinariesFromManifest(pluginManifest *cli.Manifes } return numberOfPluginPackages * len(cli.AllOSArch) } + +// IsRequiredOSArch returns true if 'osArch' is one of the require OSArch +// combinations that a plugin must support. +func IsRequiredOSArch(osArch cli.Arch) bool { + for _, requiredOSArch := range cli.MinOSArch { + if osArch == requiredOSArch { + return true + } + } + return false +} diff --git a/cmd/plugin/builder/inventory/inventory_plugin.go b/cmd/plugin/builder/inventory/inventory_plugin.go index d597a3024..f11cb0d93 100644 --- a/cmd/plugin/builder/inventory/inventory_plugin.go +++ b/cmd/plugin/builder/inventory/inventory_plugin.go @@ -109,7 +109,7 @@ func (ipuo *InventoryPluginUpdateOptions) preparePluginInventoryEntriesFromManif for i := range pluginManifest.Plugins { var pluginInventoryEntry *plugininventory.PluginInventoryEntry - for _, osArch := range cli.MinOSArch { + for _, osArch := range cli.AllOSArch { for _, version := range pluginManifest.Plugins[i].Versions { pluginInventoryEntry, err = ipuo.updatePluginInventoryEntry(pluginInventoryEntry, pluginManifest.Plugins[i], osArch, version, pluginBinaryDigestMap) if err != nil { @@ -134,7 +134,7 @@ func (ipuo *InventoryPluginUpdateOptions) fetchPluginBinaryDigest(pluginManifest fatalErrors := make(chan helpers.ErrInfo, helpers.GetNumberOfIndividualPluginBinariesFromManifest(pluginManifest)) var mutex = &sync.RWMutex{} - fetchPluginBinaryDigestFromImage := func(threadID string, pluginImage string, filename string) { + fetchPluginBinaryDigestFromImage := func(threadID string, pluginImage string, filename string, osArch cli.Arch) { defer func() { <-guard wg.Done() @@ -144,23 +144,37 @@ func (ipuo *InventoryPluginUpdateOptions) fetchPluginBinaryDigest(pluginManifest digest, err := ipuo.ImageOperationsImpl.GetFileDigestFromImage(pluginImage, filename) if err != nil { - fatalErrors <- helpers.ErrInfo{Err: errors.Wrapf(err, "error while getting plugin binary digest from the image %q", pluginImage), ID: threadID, Path: pluginImage} + // Return an error only for require OSArch combinations. + // For optional OSArch combinations, we can just ignore the missing plugin binary + // by not inserting the key to this pluginImage in the pluginBinaryDigestMap. + if helpers.IsRequiredOSArch(osArch) { + fatalErrors <- helpers.ErrInfo{Err: errors.Wrapf(err, "error while getting plugin binary digest from the image %q", pluginImage), ID: threadID, Path: pluginImage} + + // Store an empty digest to differentiate between an optional OSArch that can be ignored + // and a required OSArch that is missing a digest due to an error + mutex.Lock() + pluginBinaryDigestMap[pluginImage] = "" + mutex.Unlock() + } else { + log.Infof("%s ignoring unavailable plugin for optional os/arch: %s", threadID, osArch.String()) + } + } else { + mutex.Lock() + pluginBinaryDigestMap[pluginImage] = digest + mutex.Unlock() } - mutex.Lock() - pluginBinaryDigestMap[pluginImage] = digest - mutex.Unlock() } if !ipuo.ValidateOnly { id := 0 for i := range pluginManifest.Plugins { - for _, osArch := range cli.MinOSArch { + for _, osArch := range cli.AllOSArch { for _, version := range pluginManifest.Plugins[i].Versions { wg.Add(1) guard <- struct{}{} pluginImageBasePath := fmt.Sprintf("%s/%s/%s/%s/%s/%s:%s", ipuo.Vendor, ipuo.Publisher, osArch.OS(), osArch.Arch(), pluginManifest.Plugins[i].Target, pluginManifest.Plugins[i].Name, version) pluginImage := fmt.Sprintf("%s/%s", ipuo.Repository, pluginImageBasePath) - go fetchPluginBinaryDigestFromImage(helpers.GetID(id), pluginImage, cli.MakeArtifactName(pluginManifest.Plugins[i].Name, osArch)) + go fetchPluginBinaryDigestFromImage(helpers.GetID(id), pluginImage, cli.MakeArtifactName(pluginManifest.Plugins[i].Name, osArch), osArch) id++ } } @@ -184,13 +198,19 @@ func (ipuo *InventoryPluginUpdateOptions) fetchPluginBinaryDigest(pluginManifest // Pass the digest map to this function to update the plugin inventory entry in sync operation func (ipuo *InventoryPluginUpdateOptions) updatePluginInventoryEntry(pluginInventoryEntry *plugininventory.PluginInventoryEntry, plugin cli.Plugin, osArch cli.Arch, version string, pluginBinaryDigestMap map[string]string) (*plugininventory.PluginInventoryEntry, error) { var digest string + var exists bool pluginImageBasePath := fmt.Sprintf("%s/%s/%s/%s/%s/%s:%s", ipuo.Vendor, ipuo.Publisher, osArch.OS(), osArch.Arch(), plugin.Target, plugin.Name, version) if !ipuo.ValidateOnly { // If we are only validating the plugin's existence, we don't need to waste // resources downloading the image to get the digest which won't actually be used. pluginImage := fmt.Sprintf("%s/%s", ipuo.Repository, pluginImageBasePath) - digest = pluginBinaryDigestMap[pluginImage] + digest, exists = pluginBinaryDigestMap[pluginImage] + if !exists { + // For optional OSArch combinations, we can just ignore the missing plugin binary + return pluginInventoryEntry, nil + } + // For required OSArch, an empty digest is not expected if digest == "" { return nil, errors.Errorf("plugin binary digest cannot be empty for image %q", pluginImage) } @@ -207,7 +227,7 @@ func (ipuo *InventoryPluginUpdateOptions) updatePluginInventoryEntry(pluginInven Hidden: ipuo.DeactivatePlugins, } } - _, exists := pluginInventoryEntry.Artifacts[version] + _, exists = pluginInventoryEntry.Artifacts[version] if !exists { pluginInventoryEntry.Artifacts[version] = make([]distribution.Artifact, 0) } diff --git a/cmd/plugin/builder/template/plugintemplates/plugin-tooling.mk.tmpl b/cmd/plugin/builder/template/plugintemplates/plugin-tooling.mk.tmpl index d7515691d..0bdca2fdb 100644 --- a/cmd/plugin/builder/template/plugintemplates/plugin-tooling.mk.tmpl +++ b/cmd/plugin/builder/template/plugintemplates/plugin-tooling.mk.tmpl @@ -23,7 +23,7 @@ PLUGIN_LD_FLAGS += -X 'github.com/vmware-tanzu/tanzu-plugin-runtime/plugin/build PLUGIN_GO_FLAGS ?= # Add supported OS-ARCHITECTURE combinations here -PLUGIN_BUILD_OS_ARCH ?= linux-amd64 windows-amd64 darwin-amd64 +PLUGIN_BUILD_OS_ARCH ?= linux-amd64 windows-amd64 darwin-amd64 darwin-arm64 linux-arm64 # Paths and Directory information ROOT_DIR := $(shell git rev-parse --show-toplevel) diff --git a/plugin-tooling.mk b/plugin-tooling.mk index a66ea7ea1..961d046c9 100644 --- a/plugin-tooling.mk +++ b/plugin-tooling.mk @@ -27,7 +27,7 @@ PLUGIN_LD_FLAGS += -X 'github.com/vmware-tanzu/tanzu-plugin-runtime/plugin/build PLUGIN_GO_FLAGS ?= # Add supported OS-ARCHITECTURE combinations here -PLUGIN_BUILD_OS_ARCH ?= linux-amd64 windows-amd64 darwin-amd64 +PLUGIN_BUILD_OS_ARCH ?= linux-amd64 windows-amd64 darwin-amd64 darwin-arm64 linux-arm64 # Paths and Directory information ROOT_DIR := $(shell git rev-parse --show-toplevel)