From 13a0fe7bfe1bc2fc6ab88ee78a2d0cca8d829fb2 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Thu, 8 Aug 2024 15:08:02 +0400 Subject: [PATCH 1/4] Do not overwrite HMC values passed with the initial installation Fixes: https://github.com/Mirantis/hmc/issues/137 --- Makefile | 6 +-- api/v1alpha1/management_types.go | 11 +++++- cmd/main.go | 1 + config/dev/hmc_values.yaml | 1 - config/dev/management.yaml | 15 -------- go.mod | 2 +- internal/controller/management_controller.go | 16 ++------ internal/controller/release_controller.go | 38 ++++++++++++++++++- .../hmc/templates/rbac/rolebindings.yaml | 30 +++++++++++++++ .../rbac/{manager-rbac.yaml => roles.yaml} | 21 +++++----- 10 files changed, 94 insertions(+), 47 deletions(-) delete mode 100644 config/dev/management.yaml create mode 100644 templates/hmc/templates/rbac/rolebindings.yaml rename templates/hmc/templates/rbac/{manager-rbac.yaml => roles.yaml} (85%) diff --git a/Makefile b/Makefile index 15cc78829..934a2c01f 100644 --- a/Makefile +++ b/Makefile @@ -265,16 +265,12 @@ dev-push: docker-build helm-push dev-templates: templates-generate $(KUBECTL) -n $(NAMESPACE) apply -f templates/hmc-templates/files/templates -.PHONY: dev-management -dev-management: yq - $(YQ) '.spec.core.hmc.config += (load("config/dev/hmc_values.yaml"))' config/dev/management.yaml | $(KUBECTL) -n $(NAMESPACE) apply -f - - .PHONY: dev-aws dev-aws: yq @$(YQ) e ".data.credentials = \"${AWS_CREDENTIALS}\"" config/dev/awscredentials.yaml | $(KUBECTL) -n $(NAMESPACE) apply -f - .PHONY: dev-apply -dev-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates dev-management dev-aws +dev-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates dev-aws .PHONY: dev-destroy dev-destroy: kind-undeploy registry-undeploy diff --git a/api/v1alpha1/management_types.go b/api/v1alpha1/management_types.go index 8734f6bc7..f476ac7d9 100644 --- a/api/v1alpha1/management_types.go +++ b/api/v1alpha1/management_types.go @@ -32,6 +32,15 @@ const ( ManagementNamespace = "hmc-system" ) +var DefaultCoreConfiguration = Core{ + HMC: Component{ + Template: DefaultCoreHMCTemplate, + }, + CAPI: Component{ + Template: DefaultCoreCAPITemplate, + }, +} + // ManagementSpec defines the desired state of Management type ManagementSpec struct { // Core holds the core Management components that are mandatory. @@ -68,7 +77,7 @@ func (in *Component) HelmValues() (values map[string]interface{}, err error) { return values, err } -func (m *ManagementSpec) SetDefaults() { +func (m *ManagementSpec) SetProvidersDefaults() { m.Providers = []Component{ { Template: "k0smotron", diff --git a/cmd/main.go b/cmd/main.go index fb43c6d06..44aa4b3d8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -185,6 +185,7 @@ func main() { } if err = mgr.Add(&controller.Poller{ Client: mgr.GetClient(), + Config: mgr.GetConfig(), CreateManagement: createManagement, CreateTemplates: createTemplates, DefaultOCIRegistry: defaultOCIRegistry, diff --git a/config/dev/hmc_values.yaml b/config/dev/hmc_values.yaml index ef7ac5bd4..43a05049b 100644 --- a/config/dev/hmc_values.yaml +++ b/config/dev/hmc_values.yaml @@ -5,5 +5,4 @@ controllerManager: args: - --default-oci-registry=oci://hmc-local-registry:5000/charts - --insecure-registry=true - - --create-management=false - --create-templates=false diff --git a/config/dev/management.yaml b/config/dev/management.yaml deleted file mode 100644 index 21ddf5a59..000000000 --- a/config/dev/management.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: hmc.mirantis.com/v1alpha1 -kind: Management -metadata: - name: hmc -spec: - core: - hmc: - template: hmc - capi: - template: cluster-api - providers: - - template: k0smotron - - template: cluster-api-provider-aws - config: - credentialsSecretName: aws-credentials diff --git a/go.mod b/go.mod index 7fe02b614..a52a4091e 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/onsi/ginkgo/v2 v2.19.1 github.com/onsi/gomega v1.34.1 github.com/opencontainers/go-digest v1.0.1-0.20231025023718-d50d2fec9c98 + github.com/pkg/errors v0.9.1 github.com/segmentio/analytics-go v3.1.0+incompatible helm.sh/helm/v3 v3.15.3 k8s.io/api v0.30.3 @@ -111,7 +112,6 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/image-spec v1.1.0-rc6 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.19.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.53.0 // indirect diff --git a/internal/controller/management_controller.go b/internal/controller/management_controller.go index f0b5239a2..6cb4502e4 100644 --- a/internal/controller/management_controller.go +++ b/internal/controller/management_controller.go @@ -144,9 +144,9 @@ func wrappedComponents(mgmt *hmc.Management) (components []component) { func (r *ManagementReconciler) enableAdmissionWebhook(ctx context.Context, mgmt *hmc.Management) error { l := log.FromContext(ctx) - mgmtComponent := mgmt.Spec.Core.HMC + hmcComponent := mgmt.Spec.Core.HMC config := map[string]interface{}{} - err := json.Unmarshal(mgmtComponent.Config.Raw, &config) + err := json.Unmarshal(hmcComponent.Config.Raw, &config) if err != nil { return fmt.Errorf("failed to unmarshal HMC config into map[string]interface{}: %v", err) } @@ -167,7 +167,7 @@ func (r *ManagementReconciler) enableAdmissionWebhook(ctx context.Context, mgmt if err != nil { return fmt.Errorf("failed to marshal HMC config: %v", err) } - mgmtComponent.Config.Raw = updatedConfig + hmcComponent.Config.Raw = updatedConfig return nil } @@ -176,15 +176,7 @@ func applyDefaultCoreConfiguration(mgmt *hmc.Management) (changed bool) { // Only apply defaults when there's no configuration provided return false } - mgmt.Spec.Core = &hmc.Core{ - HMC: hmc.Component{ - Template: hmc.DefaultCoreHMCTemplate, - }, - CAPI: hmc.Component{ - Template: hmc.DefaultCoreCAPITemplate, - }, - } - + mgmt.Spec.Core = &hmc.DefaultCoreConfiguration return true } diff --git a/internal/controller/release_controller.go b/internal/controller/release_controller.go index 9344f62a8..b79e7ab9d 100644 --- a/internal/controller/release_controller.go +++ b/internal/controller/release_controller.go @@ -16,6 +16,7 @@ package controller import ( "context" + "encoding/json" "fmt" "time" @@ -25,8 +26,13 @@ import ( hcv2 "github.com/fluxcd/helm-controller/api/v2" "github.com/fluxcd/pkg/apis/meta" sourcev1 "github.com/fluxcd/source-controller/api/v1" + "github.com/pkg/errors" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/storage/driver" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -44,6 +50,8 @@ const ( type Poller struct { client.Client + Config *rest.Config + CreateManagement bool CreateTemplates bool @@ -113,8 +121,34 @@ func (p *Poller) ensureManagement(ctx context.Context) error { if !apierrors.IsNotFound(err) { return fmt.Errorf("failed to get %s/%s Management object", hmc.ManagementNamespace, hmc.ManagementName) } - mgmtObj.Spec.SetDefaults() - err := p.Create(ctx, mgmtObj) + mgmtObj.Spec.SetProvidersDefaults() + + getter := helm.NewMemoryRESTClientGetter(p.Config, p.RESTMapper()) + actionConfig := new(action.Configuration) + err = actionConfig.Init(getter, hmc.TemplatesNamespace, "secret", l.Info) + if err != nil { + return err + } + release, err := actionConfig.Releases.Last("hmc") + if err != nil { + if !errors.Is(err, driver.ErrReleaseNotFound) { + return err + } + } else { + if len(release.Config) > 0 { + values, err := json.Marshal(release.Config) + if err != nil { + return err + } + _ = applyDefaultCoreConfiguration(mgmtObj) + mgmtObj.Spec.Core = &hmc.DefaultCoreConfiguration + mgmtObj.Spec.Core.HMC.Config = &apiextensionsv1.JSON{ + Raw: values, + } + } + } + + err = p.Create(ctx, mgmtObj) if err != nil { return fmt.Errorf("failed to create %s/%s Management object", hmc.ManagementNamespace, hmc.ManagementName) } diff --git a/templates/hmc/templates/rbac/rolebindings.yaml b/templates/hmc/templates/rbac/rolebindings.yaml new file mode 100644 index 000000000..d510c533e --- /dev/null +++ b/templates/hmc/templates/rbac/rolebindings.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hmc.fullname" . }}-manager-rolebinding + labels: + {{- include "hmc.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: '{{ include "hmc.fullname" . }}-manager-role' +subjects: +- kind: ServiceAccount + name: '{{ include "hmc.fullname" . }}-controller-manager' + namespace: '{{ .Release.Namespace }}' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hmc.fullname" . }}-manager-secrets-reader-rolebinding + namespace: {{ .Release.Namespace }} + labels: + {{- include "hmc.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: '{{ include "hmc.fullname" . }}-manager-secrets-reader-role' +subjects: + - kind: ServiceAccount + name: '{{ include "hmc.fullname" . }}-controller-manager' + namespace: '{{ .Release.Namespace }}' diff --git a/templates/hmc/templates/rbac/manager-rbac.yaml b/templates/hmc/templates/rbac/roles.yaml similarity index 85% rename from templates/hmc/templates/rbac/manager-rbac.yaml rename to templates/hmc/templates/rbac/roles.yaml index 79d0675b4..7cad5c226 100644 --- a/templates/hmc/templates/rbac/manager-rbac.yaml +++ b/templates/hmc/templates/rbac/roles.yaml @@ -142,16 +142,17 @@ rules: - create --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: Role metadata: - name: {{ include "hmc.fullname" . }}-manager-rolebinding + name: {{ include "hmc.fullname" . }}-manager-secrets-reader-role + namespace: {{ .Release.Namespace }} labels: {{- include "hmc.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "hmc.fullname" . }}-manager-role' -subjects: -- kind: ServiceAccount - name: '{{ include "hmc.fullname" . }}-controller-manager' - namespace: '{{ .Release.Namespace }}' +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list From 03d8e206118efbe68e29f6256bc00752b7e5667a Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Thu, 8 Aug 2024 15:18:50 +0400 Subject: [PATCH 2/4] Add a note about HMC customization limitations --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4b5b24a9c..a7fd70fe2 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ or install using `helm` Then follow the [Deploy a managed cluster](#deploy-a-managed-cluster) guide to create a managed cluster. +> Note. The HMC installation using Kubernetes manifests doesn't allow customizing the +> deployment. If the custom HMC configuration should be applied, install HMC using +> the Helm chart. + ### Development guide See [Install HMC for development purposes](docs/dev.md#hmc-installation-for-development). From 791b99c3678110eee4bba1377397a746c7b8da38 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova <41469478+eromanova@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:06:28 +0400 Subject: [PATCH 3/4] Update README.md Co-authored-by: Andrey Pavlov Signed-off-by: Ekaterina Kazakova <41469478+eromanova@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a7fd70fe2..85e53f6cd 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ or install using `helm` Then follow the [Deploy a managed cluster](#deploy-a-managed-cluster) guide to create a managed cluster. -> Note. The HMC installation using Kubernetes manifests doesn't allow customizing the +> Note: The HMC installation using Kubernetes manifests does not allow customization of the deployment. To apply a custom HMC configuration, install HMC using the Helm chart. > deployment. If the custom HMC configuration should be applied, install HMC using > the Helm chart. From 6050bc9e33e73ce68bf5cac1fc3ef2a707f224d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Thu, 8 Aug 2024 16:09:21 +0400 Subject: [PATCH 4/4] Fix panic in case no custom config provided --- internal/controller/management_controller.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/controller/management_controller.go b/internal/controller/management_controller.go index 6cb4502e4..ac94d790d 100644 --- a/internal/controller/management_controller.go +++ b/internal/controller/management_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/fluxcd/pkg/apis/meta" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -144,18 +145,21 @@ func wrappedComponents(mgmt *hmc.Management) (components []component) { func (r *ManagementReconciler) enableAdmissionWebhook(ctx context.Context, mgmt *hmc.Management) error { l := log.FromContext(ctx) - hmcComponent := mgmt.Spec.Core.HMC - config := map[string]interface{}{} - err := json.Unmarshal(hmcComponent.Config.Raw, &config) - if err != nil { - return fmt.Errorf("failed to unmarshal HMC config into map[string]interface{}: %v", err) + hmcComponent := &mgmt.Spec.Core.HMC + config := make(map[string]interface{}) + + if hmcComponent.Config != nil { + err := json.Unmarshal(hmcComponent.Config.Raw, &config) + if err != nil { + return fmt.Errorf("failed to unmarshal HMC config into map[string]interface{}: %v", err) + } } admissionWebhookValues := make(map[string]interface{}) if config["admissionWebhook"] != nil { admissionWebhookValues = config["admissionWebhook"].(map[string]interface{}) } - err = certmanager.VerifyAPI(ctx, r.Config, r.Scheme, hmc.ManagementNamespace) + err := certmanager.VerifyAPI(ctx, r.Config, r.Scheme, hmc.ManagementNamespace) if err != nil { return fmt.Errorf("failed to check in the cert-manager API is installed: %v", err) } @@ -167,7 +171,9 @@ func (r *ManagementReconciler) enableAdmissionWebhook(ctx context.Context, mgmt if err != nil { return fmt.Errorf("failed to marshal HMC config: %v", err) } - hmcComponent.Config.Raw = updatedConfig + hmcComponent.Config = &apiextensionsv1.JSON{ + Raw: updatedConfig, + } return nil }