From 877f9ab6676c28b8f230cff1c1d5c51016805382 Mon Sep 17 00:00:00 2001 From: Lin Yang Date: Tue, 5 Sep 2023 16:51:16 +0800 Subject: [PATCH] feat: add cli commands to enable/disable ingress, gateway, egress-gateway, flb and service-lb (#39) * feat: enable fsm-ingress by cli cmd Signed-off-by: Lin Yang * feat: add --dry-run flag for install command Signed-off-by: Lin Yang feat: add --dry-run flag for install command Signed-off-by: Lin Yang * feat: add commands for enable/disable ingress Signed-off-by: Lin Yang * fix: command description Signed-off-by: Lin Yang * fix: golang lint Signed-off-by: Lin Yang * feat: support enable/disable gateway & NamespacedIngress Signed-off-by: Lin Yang * feat: cleanup resources upon uninstallation Signed-off-by: Lin Yang * feat: support enable/disable egress-gateway Signed-off-by: Lin Yang * #32 rollback change for balance algorithm shortening (#33) * feat: support enable/disable service-lb Signed-off-by: Lin Yang * fix: comments Signed-off-by: Lin Yang * feat: waiting for fsm-controller pods to be ready Signed-off-by: Lin Yang * fix: a typo Signed-off-by: Lin Yang * feat: waiting for fsm-controller to ready Signed-off-by: Lin Yang * fix: golang lint Signed-off-by: Lin Yang * feat: wait for deployment rolled out Signed-off-by: Lin Yang * ci: watch branch release/v* to trigger GitHub actions (#38) Signed-off-by: Lin Yang * fix: make codegen Signed-off-by: Lin Yang * fix: remove cluster argument Signed-off-by: Lin Yang --------- Signed-off-by: Lin Yang Co-authored-by: Addo.Zhang --- charts/fsm/templates/cleanup-hook.yaml | 28 +- .../templates/egress-gateway-configmap.yaml | 4 +- .../templates/egress-gateway-deployment.yaml | 34 +- .../fsm/templates/egress-gateway-service.yaml | 8 +- charts/fsm/templates/fsm-ingress-class.yaml | 1 + .../fsm/templates/fsm-ingress-deployment.yaml | 1 + charts/fsm/templates/fsm-ingress-service.yaml | 1 + charts/fsm/templates/preset-mesh-config.yaml | 12 + cmd/cli/cluster.go | 1 + cmd/cli/egressgateway.go | 36 ++ cmd/cli/egressgateway_disable.go | 108 +++++ cmd/cli/egressgateway_enable.go | 218 +++++++++ cmd/cli/flb.go | 34 ++ cmd/cli/flb_disable.go | 104 +++++ cmd/cli/flb_enable.go | 200 ++++++++ cmd/cli/fsm.go | 5 + cmd/cli/gateway.go | 26 ++ cmd/cli/gateway_disable.go | 121 +++++ cmd/cli/gateway_enable.go | 137 ++++++ cmd/cli/ingress.go | 37 ++ cmd/cli/ingress_disable.go | 116 +++++ cmd/cli/ingress_enable.go | 283 ++++++++++++ cmd/cli/ingress_namespaced.go | 21 + cmd/cli/ingress_namespaced_disable.go | 123 +++++ cmd/cli/ingress_namespaced_enable.go | 132 ++++++ cmd/cli/install.go | 14 + cmd/cli/namespacedingress.go | 1 + cmd/cli/servicelb.go | 26 ++ cmd/cli/servicelb_disable.go | 109 +++++ cmd/cli/servicelb_enable.go | 106 +++++ cmd/cli/shared.go | 26 ++ cmd/cli/util.go | 427 ++++++++++++++++++ .../crds/config.flomesh.io_meshconfigs.yaml | 13 +- pkg/apis/config/v1alpha3/mesh_config.go | 52 ++- .../config/v1alpha3/zz_generated.deepcopy.go | 49 ++ pkg/constants/constants.go | 6 + .../gateway/v1beta1/gateway_controller.go | 19 +- .../v1alpha1/namespacedingress_controller.go | 19 +- .../servicelb/service_controller.go | 13 +- pkg/gateway/client.go | 2 +- pkg/helm/helm.go | 151 +++++-- pkg/helm/types.go | 9 +- pkg/sidecar/providers/pipy/repo/plugin.go | 2 +- pkg/utils/ctrl.go | 121 ++++- pkg/webhook/gateway/gateway_webhook.go | 46 +- .../namespacedingress_webhook.go | 15 +- 46 files changed, 2926 insertions(+), 91 deletions(-) create mode 100644 cmd/cli/cluster.go create mode 100644 cmd/cli/egressgateway.go create mode 100644 cmd/cli/egressgateway_disable.go create mode 100644 cmd/cli/egressgateway_enable.go create mode 100644 cmd/cli/flb.go create mode 100644 cmd/cli/flb_disable.go create mode 100644 cmd/cli/flb_enable.go create mode 100644 cmd/cli/gateway.go create mode 100644 cmd/cli/gateway_disable.go create mode 100644 cmd/cli/gateway_enable.go create mode 100644 cmd/cli/ingress.go create mode 100644 cmd/cli/ingress_disable.go create mode 100644 cmd/cli/ingress_enable.go create mode 100644 cmd/cli/ingress_namespaced.go create mode 100644 cmd/cli/ingress_namespaced_disable.go create mode 100644 cmd/cli/ingress_namespaced_enable.go create mode 100644 cmd/cli/namespacedingress.go create mode 100644 cmd/cli/servicelb.go create mode 100644 cmd/cli/servicelb_disable.go create mode 100644 cmd/cli/servicelb_enable.go create mode 100644 cmd/cli/shared.go diff --git a/charts/fsm/templates/cleanup-hook.yaml b/charts/fsm/templates/cleanup-hook.yaml index 1ee320f89..2a2c2fbf9 100644 --- a/charts/fsm/templates/cleanup-hook.yaml +++ b/charts/fsm/templates/cleanup-hook.yaml @@ -12,6 +12,18 @@ rules: - apiGroups: ["config.flomesh.io"] resources: ["meshconfigs"] verbs: ["delete"] + - apiGroups: [ "" ] + resources: [ "services", "configmaps" ] + verbs: [ "delete" ] + - apiGroups: [ "app" ] + resources: [ "deployments" ] + verbs: [ "delete" ] + - apiGroups: [ "networking.k8s.io" ] + resources: [ "ingressclasses" ] + verbs: [ "delete" ] + - apiGroups: [ "app" ] + resources: [ "daemonsets" ] + verbs: [ "get", "list", "create", "update", "patch", "delete" ] - apiGroups: [ "" ] resources: [ "secrets"] verbs: ["get", "list", "create", "update", "patch", "delete"] @@ -22,7 +34,10 @@ rules: resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] verbs: ["get", "list", "create", "update", "patch", "delete"] - apiGroups: [ "gateway.networking.k8s.io" ] - resources: [ "gatewayclasses" ] + resources: [ "gatewayclasses", "gateways" ] + verbs: [ "get", "list", "create", "update", "patch", "delete" ] + - apiGroups: [ "flomesh.io" ] + resources: [ "namespacedingresses" ] verbs: [ "get", "list", "create", "update", "patch", "delete" ] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -92,7 +107,16 @@ spec: kubectl delete --ignore-not-found meshrootcertificate -n '{{ include "fsm.namespace" . }}' fsm-mesh-root-certificate; kubectl delete mutatingwebhookconfiguration -l app.kubernetes.io/name=flomesh.io,app.kubernetes.io/instance={{ .Values.fsm.meshName }},app.kubernetes.io/version={{ .Chart.AppVersion }},app=fsm-injector --ignore-not-found; kubectl delete validatingwebhookconfiguration -l app.kubernetes.io/name=flomesh.io,app.kubernetes.io/instance={{ .Values.fsm.meshName }},app.kubernetes.io/version={{ .Chart.AppVersion }},app=fsm-controller --ignore-not-found; - kubectl delete gatewayclasses.gateway.networking.k8s.io -l app.kubernetes.io/name=flomesh.io,app.kubernetes.io/instance={{ .Values.fsm.meshName }},app.kubernetes.io/version={{ .Chart.AppVersion }},app=fsm-controller --ignore-not-found; + kubectl delete gatewayclasses.gateway.networking.k8s.io -l app.kubernetes.io/name=flomesh.io,app.kubernetes.io/instance={{ .Values.fsm.meshName }},app.kubernetes.io/version={{ .Chart.AppVersion }},app=fsm-gateway --ignore-not-found; + kubectl delete gateways.gateway.networking.k8s.io -l app.kubernetes.io/name=flomesh.io,app.kubernetes.io/instance={{ .Values.fsm.meshName }},app.kubernetes.io/version={{ .Chart.AppVersion }},app=fsm-gateway --ignore-not-found; + kubectl delete namespacedingresses.flomesh.io -l app.kubernetes.io/name=flomesh.io,app.kubernetes.io/instance={{ .Values.fsm.meshName }},app.kubernetes.io/version={{ .Chart.AppVersion }},app=fsm-ingress --ignore-not-found; + kubectl delete daemonsets -l app.kubernetes.io/name=flomesh.io,app.kubernetes.io/instance={{ .Values.fsm.meshName }},app.kubernetes.io/version={{ .Chart.AppVersion }},app=fsm-servicelb --ignore-not-found; + kubectl delete ingressclasses pipy --ignore-not-found; + kubectl delete deploy fsm-ingress -n '{{ include "fsm.namespace" . }}' --ignore-not-found; + kubectl delete svc fsm-ingress -n '{{ include "fsm.namespace" . }}' --ignore-not-found; + kubectl delete deploy fsm-egress-gateway -n '{{ include "fsm.namespace" . }}' --ignore-not-found; + kubectl delete svc fsm-egress-gateway -n '{{ include "fsm.namespace" . }}' --ignore-not-found; + kubectl delete cm fsm-egress-gateway-pjs -n '{{ include "fsm.namespace" . }}' --ignore-not-found; {{- if .Values.fsm.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.fsm.imagePullSecrets | indent 8 }} diff --git a/charts/fsm/templates/egress-gateway-configmap.yaml b/charts/fsm/templates/egress-gateway-configmap.yaml index fc4e828cc..555a8f782 100644 --- a/charts/fsm/templates/egress-gateway-configmap.yaml +++ b/charts/fsm/templates/egress-gateway-configmap.yaml @@ -5,7 +5,9 @@ metadata: name: fsm-egress-gateway-pjs namespace: {{ include "fsm.namespace" . }} labels: - {{- include "fsm.egress-gateway.labels" . | nindent 4 }} + {{- include "fsm.labels" . | nindent 4 }} + app: fsm-egress-gateway + meshName: {{ .Values.fsm.meshName }} data: egress-gateway.js: | {{- if eq .Values.fsm.egressGateway.mode "sock5" }} diff --git a/charts/fsm/templates/egress-gateway-deployment.yaml b/charts/fsm/templates/egress-gateway-deployment.yaml index b3008927b..cc8bc9826 100644 --- a/charts/fsm/templates/egress-gateway-deployment.yaml +++ b/charts/fsm/templates/egress-gateway-deployment.yaml @@ -5,20 +5,21 @@ metadata: name: {{ .Values.fsm.egressGateway.name }} namespace: {{ include "fsm.namespace" . }} labels: - {{- include "fsm.egress-gateway.labels" . | nindent 4 }} - {{- include "fsm.egress-gateway.selectorLabels" . | nindent 4 }} + {{- include "fsm.labels" . | nindent 4 }} + app: fsm-egress-gateway + meshName: {{ .Values.fsm.meshName }} spec: replicas: {{ .Values.fsm.egressGateway.replicaCount }} selector: matchLabels: - {{- include "fsm.egress-gateway.selectorLabels" . | nindent 6 }} + app: fsm-egress-gateway strategy: type: RollingUpdate template: metadata: labels: - {{- include "fsm.egress-gateway.labels" . | nindent 8 }} - {{- include "fsm.egress-gateway.selectorLabels" . | nindent 8 }} + {{- include "fsm.labels" . | nindent 8 }} + app: fsm-egress-gateway {{- with .Values.fsm.egressGateway.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -32,12 +33,12 @@ spec: spec: containers: - name: pipy - image: {{ include "fsm.pipy-repo.image" . }} + image: {{ .Values.fsm.repoServer.image }} imagePullPolicy: {{ .Values.fsm.image.pullPolicy }} resources: {{- toYaml .Values.fsm.egressGateway.resources | nindent 10 }} volumeMounts: - - name: {{ .Values.fsm.configmaps.egress.name }} + - name: fsm-egress-gateway-pjs mountPath: "/repo/egress-gateway.js" subPath: egress-gateway.js readOnly: true @@ -53,13 +54,22 @@ spec: - "--log-level={{ .Values.fsm.egressGateway.logLevel }}" - "--admin-port={{ .Values.fsm.egressGateway.adminPort }}" env: - {{- include "fsm.common-env" . | nindent 10 }} + - name: FSM_NAMESPACE + value: {{ include "fsm.namespace" . }} + - name: FSM_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: FSM_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumes: - - name: {{ .Values.fsm.configmaps.egress.name }} + - name: fsm-egress-gateway-pjs configMap: - name: {{ .Values.fsm.configmaps.egress.name }} - serviceAccountName: {{ include "fsm.serviceAccountName" . }} - {{- with .Values.fsm.image.pullSecrets }} + name: fsm-egress-gateway-pjs + serviceAccountName: {{ .Release.Name }} + {{- with .Values.fsm.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/fsm/templates/egress-gateway-service.yaml b/charts/fsm/templates/egress-gateway-service.yaml index 5e84d1cb0..9ebb86614 100644 --- a/charts/fsm/templates/egress-gateway-service.yaml +++ b/charts/fsm/templates/egress-gateway-service.yaml @@ -5,9 +5,11 @@ metadata: name: fsm-egress-gateway namespace: {{ include "fsm.namespace" . }} labels: - {{- include "fsm.egress-gateway.labels" . | nindent 4 }} + {{- include "fsm.labels" . | nindent 4 }} + app: fsm-egress-gateway + meshName: {{ .Values.fsm.meshName }} annotations: - {{- include "fsm.egress-gateway.annotations" . | nindent 4 }} + flomesh.io/egress-gateway-mode: {{ .Values.fsm.egressGateway.mode }} spec: ports: - port: {{ .Values.fsm.egressGateway.port }} @@ -16,5 +18,5 @@ spec: protocol: TCP appProtocol: tcp selector: - {{- include "fsm.egress-gateway.selectorLabels" . | nindent 4 }} + app: fsm-egress-gateway {{- end }} \ No newline at end of file diff --git a/charts/fsm/templates/fsm-ingress-class.yaml b/charts/fsm/templates/fsm-ingress-class.yaml index 00ead342d..153a50ef6 100644 --- a/charts/fsm/templates/fsm-ingress-class.yaml +++ b/charts/fsm/templates/fsm-ingress-class.yaml @@ -5,6 +5,7 @@ metadata: name: pipy labels: {{- include "fsm.labels" . | nindent 4 }} + app: fsm-ingress annotations: meta.flomesh.io/namespace: {{ include "fsm.namespace" . }} meta.flomesh.io/ingress-pipy-svc: "fsm-ingress" diff --git a/charts/fsm/templates/fsm-ingress-deployment.yaml b/charts/fsm/templates/fsm-ingress-deployment.yaml index e74ab61f4..edf5cd1cc 100644 --- a/charts/fsm/templates/fsm-ingress-deployment.yaml +++ b/charts/fsm/templates/fsm-ingress-deployment.yaml @@ -9,6 +9,7 @@ metadata: {{- include "fsm.labels" . | nindent 4 }} app: fsm-ingress meshName: {{ .Values.fsm.meshName }} + ingress.flomesh.io/namespaced: "false" spec: replicas: {{ .Values.fsm.fsmIngress.replicaCount }} selector: diff --git a/charts/fsm/templates/fsm-ingress-service.yaml b/charts/fsm/templates/fsm-ingress-service.yaml index 4f8c112de..a3f3f3b10 100644 --- a/charts/fsm/templates/fsm-ingress-service.yaml +++ b/charts/fsm/templates/fsm-ingress-service.yaml @@ -8,6 +8,7 @@ metadata: labels: {{- include "fsm.labels" . | nindent 4 }} app: fsm-ingress + meshName: {{ .Values.fsm.meshName }} ingress.flomesh.io/namespaced: "false" {{- with .Values.fsm.fsmIngress.service.annotations }} annotations: diff --git a/charts/fsm/templates/preset-mesh-config.yaml b/charts/fsm/templates/preset-mesh-config.yaml index 678893d70..614087021 100644 --- a/charts/fsm/templates/preset-mesh-config.yaml +++ b/charts/fsm/templates/preset-mesh-config.yaml @@ -120,9 +120,21 @@ data: "strictMode": {{ .Values.fsm.flb.strictMode }}, "secretName": "{{ .Values.fsm.flb.secretName }}" }, + "egressGateway": { + "enabled": {{ .Values.fsm.egressGateway.enabled }}, + "logLevel": "{{ .Values.fsm.egressGateway.logLevel }}", + "mode": "{{ .Values.fsm.egressGateway.mode }}", + "port": {{ .Values.fsm.egressGateway.port }}, + "adminPort": {{ .Values.fsm.egressGateway.adminPort }}, + "replicas": {{ .Values.fsm.egressGateway.replicaCount }} + }, "image": { "registry": "{{ .Values.fsm.image.registry }}", "tag": "{{ .Values.fsm.image.tag }}", "pullPolicy": "{{ .Values.fsm.image.pullPolicy }}" + }, + "misc": { + "curlImage": "{{ .Values.fsm.curlImage }}", + "repoServerImage": "{{ .Values.fsm.repoServer.image }}" } } diff --git a/cmd/cli/cluster.go b/cmd/cli/cluster.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/cmd/cli/cluster.go @@ -0,0 +1 @@ +package main diff --git a/cmd/cli/egressgateway.go b/cmd/cli/egressgateway.go new file mode 100644 index 000000000..1aa649a69 --- /dev/null +++ b/cmd/cli/egressgateway.go @@ -0,0 +1,36 @@ +package main + +import ( + "io" + + "helm.sh/helm/v3/pkg/action" + + "github.com/spf13/cobra" +) + +const egressGatewayDescription = ` +This command consists of multiple subcommands related to managing egress gateway +associated with fsm installations. +` + +var ( + egressGatewayManifestFiles = []string{ + "templates/egress-gateway-configmap.yaml", + "templates/egress-gateway-deployment.yaml", + "templates/egress-gateway-service.yaml", + } +) + +func newEgressGatewayCmd(config *action.Configuration, out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "egressgateway", + Short: "manage fsm egress-gateway", + Aliases: []string{"egw"}, + Long: egressGatewayDescription, + Args: cobra.NoArgs, + } + cmd.AddCommand(newEgressGatewayEnable(config, out)) + cmd.AddCommand(newEgressGatewayDisable(out)) + + return cmd +} diff --git a/cmd/cli/egressgateway_disable.go b/cmd/cli/egressgateway_disable.go new file mode 100644 index 000000000..3f01b5c78 --- /dev/null +++ b/cmd/cli/egressgateway_disable.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const egressGatewayDisableDescription = ` +This command will disable FSM egress-gateway, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type egressGatewayDisableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + meshName string +} + +func newEgressGatewayDisable(out io.Writer) *cobra.Command { + disableCmd := &egressGatewayDisableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "disable", + Short: "disable fsm egress-gateway", + Long: egressGatewayDisableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.configClient = configClient + + return disableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&disableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *egressGatewayDisableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !mc.Spec.EgressGateway.Enabled { + fmt.Fprintf(cmd.out, "egress-gateway is disabled already, no action needed\n") + return nil + } + + debug("Deleting FSM egress-gateway resources ...") + err = deleteEgressGatewayResources(ctx, cmd.kubeClient, fsmNamespace, cmd.meshName) + if err != nil { + return err + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "egressGateway.enabled": false, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.EgressGateway.Enabled = false + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + fmt.Fprintf(cmd.out, "egress-gateway is disabled successfully\n") + + return nil +} diff --git a/cmd/cli/egressgateway_enable.go b/cmd/cli/egressgateway_enable.go new file mode 100644 index 000000000..272b9ca34 --- /dev/null +++ b/cmd/cli/egressgateway_enable.go @@ -0,0 +1,218 @@ +package main + +import ( + "context" + "fmt" + "io" + "time" + + "k8s.io/utils/pointer" + + "github.com/flomesh-io/fsm/pkg/constants" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha3 "github.com/flomesh-io/fsm/pkg/apis/config/v1alpha3" + + "github.com/spf13/cobra" + "helm.sh/helm/v3/pkg/action" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const egressGatewayEnableDescription = ` +This command will enable FSM egress-gateway, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type egressGatewayEnableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + dynamicClient dynamic.Interface + configClient configClientset.Interface + mapper meta.RESTMapper + actionConfig *action.Configuration + meshName string + mode string + logLevel string + adminPort int32 + port int32 + replicas int32 +} + +func (cmd *egressGatewayEnableCmd) GetActionConfig() *action.Configuration { + return cmd.actionConfig +} + +func (cmd *egressGatewayEnableCmd) GetDynamicClient() dynamic.Interface { + return cmd.dynamicClient +} + +func (cmd *egressGatewayEnableCmd) GetRESTMapper() meta.RESTMapper { + return cmd.mapper +} + +func (cmd *egressGatewayEnableCmd) GetMeshName() string { + return cmd.meshName +} + +func newEgressGatewayEnable(actionConfig *action.Configuration, out io.Writer) *cobra.Command { + enableCmd := &egressGatewayEnableCmd{ + out: out, + actionConfig: actionConfig, + } + + cmd := &cobra.Command{ + Use: "enable", + Short: "enable fsm egress-gateway", + Long: egressGatewayEnableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.kubeClient = kubeClient + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.dynamicClient = dynamicClient + + gr, err := restmapper.GetAPIGroupResources(kubeClient.Discovery()) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + + mapper := restmapper.NewDiscoveryRESTMapper(gr) + enableCmd.mapper = mapper + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.configClient = configClient + + return enableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&enableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + f.StringVar(&enableCmd.mode, "mode", constants.EgressGatewayModeHTTP2Tunnel, "mode of the egress-gateway, http2tunnel or sock5") + f.StringVar(&enableCmd.logLevel, "log-level", "error", "log level of egress-gateway") + f.Int32Var(&enableCmd.adminPort, "admin-port", 6060, "admin port of egress-gateway, rarely need to be set manually") + f.Int32Var(&enableCmd.port, "port", 1080, "serving port of egress-gateway") + f.Int32Var(&enableCmd.replicas, "replicas", 1, "replicas of egress-gateway") + //utilruntime.Must(cmd.MarkFlagRequired("mode")) + + return cmd +} + +func (cmd *egressGatewayEnableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if cmd.mode != constants.EgressGatewayModeHTTP2Tunnel && cmd.mode != constants.EgressGatewayModeSock5 { + return fmt.Errorf("mode must be either http2tunnel or socks5") + } + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if mc.Spec.EgressGateway.Enabled { + fmt.Fprintf(cmd.out, "egress-gateway is enabled, no action needed\n") + return nil + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "egressGateway": map[string]interface{}{ + "enabled": true, + "logLevel": cmd.logLevel, + "mode": cmd.mode, + "port": cmd.port, + "adminPort": cmd.adminPort, + "replicas": cmd.replicas, + }, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.EgressGateway = configv1alpha3.EgressGatewaySpec{ + Enabled: true, + LogLevel: cmd.logLevel, + Mode: cmd.mode, + Port: pointer.Int32(cmd.port), + AdminPort: pointer.Int32(cmd.adminPort), + Replicas: pointer.Int32(cmd.replicas), + } + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := installManifests(cmd, mc, fsmNamespace, kubeVersion119, egressGatewayManifestFiles...); err != nil { + return err + } + + time.Sleep(3 * time.Second) + + deployment, err := cmd.kubeClient.AppsV1(). + Deployments(fsmNamespace). + Get(ctx, constants.FSMEgressGatewayName, metav1.GetOptions{}) + if err != nil { + return err + } + + if err := waitForDeploymentReady(ctx, cmd.kubeClient, deployment, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "egress-gateway is enabled successfully\n") + + return nil +} + +func (cmd *egressGatewayEnableCmd) ResolveValues(mc *configv1alpha3.MeshConfig) (map[string]interface{}, error) { + finalValues := map[string]interface{}{} + + valuesConfig := []string{ + fmt.Sprintf("fsm.egressGateway.enabled=%t", true), + fmt.Sprintf("fsm.egressGateway.logLevel=%s", cmd.logLevel), + fmt.Sprintf("fsm.egressGateway.mode=%s", cmd.mode), + fmt.Sprintf("fsm.egressGateway.port=%d", cmd.port), + fmt.Sprintf("fsm.egressGateway.adminPort=%d", cmd.adminPort), + fmt.Sprintf("fsm.egressGateway.replicas=%d", cmd.replicas), + fmt.Sprintf("fsm.fsmNamespace=%s", mc.GetNamespace()), + fmt.Sprintf("fsm.meshName=%s", cmd.meshName), + fmt.Sprintf("fsm.image.registry=%s", mc.Spec.Image.Registry), + fmt.Sprintf("fsm.image.pullPolicy=%s", mc.Spec.Image.PullPolicy), + fmt.Sprintf("fsm.image.tag=%s", mc.Spec.Image.Tag), + fmt.Sprintf("fsm.repoServer.image=%s", mc.Spec.Misc.RepoServerImage), + } + + if err := parseVal(valuesConfig, finalValues); err != nil { + return nil, err + } + + return finalValues, nil +} diff --git a/cmd/cli/flb.go b/cmd/cli/flb.go new file mode 100644 index 000000000..a9972f498 --- /dev/null +++ b/cmd/cli/flb.go @@ -0,0 +1,34 @@ +package main + +import ( + "io" + + "helm.sh/helm/v3/pkg/action" + + "github.com/spf13/cobra" +) + +const flbDescription = ` +This command consists of multiple subcommands related to managing flb controller +associated with fsm installations. +` + +var ( + flbManifestFiles = []string{ + "templates/fsm-flb-secret.yaml", + } +) + +func newFLBCmd(config *action.Configuration, out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "flb", + Short: "manage fsm FLB", + Aliases: []string{"flb"}, + Long: flbDescription, + Args: cobra.NoArgs, + } + cmd.AddCommand(newFLBEnableCmd(config, out)) + cmd.AddCommand(newFLBDisableCmd(out)) + + return cmd +} diff --git a/cmd/cli/flb_disable.go b/cmd/cli/flb_disable.go new file mode 100644 index 000000000..056fb901d --- /dev/null +++ b/cmd/cli/flb_disable.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const flbDisableDescription = ` +This command will disable FSM FLB, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type flbDisableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface +} + +func newFLBDisableCmd(out io.Writer) *cobra.Command { + disableCmd := &flbDisableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "disable", + Short: "disable fsm FLB", + Long: flbDisableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.configClient = configClient + + return disableCmd.run() + }, + } + + return cmd +} + +func (cmd *flbDisableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !mc.Spec.FLB.Enabled { + fmt.Fprintf(cmd.out, "FLB is disabled already, no action needed\n") + return nil + } + + if err := updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "flb.enabled": false, + }); err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.FLB.Enabled = false + if _, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}); err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + debug("Deleting FSM FLB resources ...") + if err := deleteFLBResources(ctx, cmd.kubeClient); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "service-lb is disabled successfully\n") + + return nil +} diff --git a/cmd/cli/flb_enable.go b/cmd/cli/flb_enable.go new file mode 100644 index 000000000..070c3cdb7 --- /dev/null +++ b/cmd/cli/flb_enable.go @@ -0,0 +1,200 @@ +package main + +import ( + "context" + "fmt" + "io" + + configv1alpha3 "github.com/flomesh-io/fsm/pkg/apis/config/v1alpha3" + + "k8s.io/client-go/restmapper" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/dynamic" + + "helm.sh/helm/v3/pkg/action" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const flbEnableDescription = ` +This command will enable FSM FLB, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type flbEnableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + dynamicClient dynamic.Interface + mapper meta.RESTMapper + actionConfig *action.Configuration + meshName string + strictMode bool + secretName string + baseUrl string + userName string + password string + k8sCluster string + addressPool string + algo string +} + +func (cmd *flbEnableCmd) GetActionConfig() *action.Configuration { + return cmd.actionConfig +} + +func (cmd *flbEnableCmd) GetDynamicClient() dynamic.Interface { + return cmd.dynamicClient +} + +func (cmd *flbEnableCmd) GetRESTMapper() meta.RESTMapper { + return cmd.mapper +} + +func (cmd *flbEnableCmd) GetMeshName() string { + return cmd.meshName +} + +func newFLBEnableCmd(config *action.Configuration, out io.Writer) *cobra.Command { + enableCmd := &flbEnableCmd{ + out: out, + actionConfig: config, + } + + cmd := &cobra.Command{ + Use: "enable", + Short: "enable fsm FLB", + Long: flbEnableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.configClient = configClient + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.dynamicClient = dynamicClient + + gr, err := restmapper.GetAPIGroupResources(kubeClient.Discovery()) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + + mapper := restmapper.NewDiscoveryRESTMapper(gr) + enableCmd.mapper = mapper + + return enableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&enableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + f.BoolVar(&enableCmd.strictMode, "strict-mode", false, "enable strict mode for FLB") + f.StringVar(&enableCmd.secretName, "secret-name", "fsm-flb-secret", "name of the secret for storing FLB config") + f.StringVar(&enableCmd.baseUrl, "base-url", "http://localhost:1337", "base URL of FLB API server") + f.StringVar(&enableCmd.userName, "username", "admin", "user name of FLB API server") + f.StringVar(&enableCmd.password, "password", "admin", "password of FLB API server") + f.StringVar(&enableCmd.k8sCluster, "k8s-cluster", "UNKNOWN", "name of the k8s cluster in which FLB controller is running") + f.StringVar(&enableCmd.addressPool, "address-pool", "default", "name of the address pool of FLB") + f.StringVar(&enableCmd.algo, "algo", "default", "load balancing algorithm of FLB") + + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *flbEnableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if mc.Spec.ServiceLB.Enabled { + fmt.Fprintf(cmd.out, "service-lb is enabled already, no action needed\n") + return nil + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "flb.enabled": true, + "flb.strictMode": cmd.strictMode, + "flb.secretName": cmd.secretName, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.FLB.Enabled = true + mc.Spec.FLB.StrictMode = cmd.strictMode + mc.Spec.FLB.SecretName = cmd.secretName + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := installManifests(cmd, mc, fsmNamespace, kubeVersion119, flbManifestFiles...); err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "FLB is enabled successfully\n") + + return nil +} + +func (cmd *flbEnableCmd) ResolveValues(mc *configv1alpha3.MeshConfig) (map[string]interface{}, error) { + finalValues := map[string]interface{}{} + + valuesConfig := []string{ + fmt.Sprintf("fsm.flb.enabled=%t", true), + fmt.Sprintf("fsm.flb.strictMode=%t", cmd.strictMode), + fmt.Sprintf("fsm.flb.secretName=%s", cmd.secretName), + fmt.Sprintf("fsm.flb.baseUrl=%s", cmd.baseUrl), + fmt.Sprintf("fsm.flb.password=%s", cmd.password), + fmt.Sprintf("fsm.flb.k8sCluster=%s", cmd.kubeClient), + fmt.Sprintf("fsm.flb.defaultAddressPool=%s", cmd.addressPool), + fmt.Sprintf("fsm.flb.defaultAlgo=%s", cmd.algo), + fmt.Sprintf("fsm.fsmNamespace=%s", mc.GetNamespace()), + fmt.Sprintf("fsm.meshName=%s", cmd.meshName), + fmt.Sprintf("fsm.image.registry=%s", mc.Spec.Image.Registry), + fmt.Sprintf("fsm.image.pullPolicy=%s", mc.Spec.Image.PullPolicy), + fmt.Sprintf("fsm.image.tag=%s", mc.Spec.Image.Tag), + } + + if err := parseVal(valuesConfig, finalValues); err != nil { + return nil, err + } + + return finalValues, nil +} diff --git a/cmd/cli/fsm.go b/cmd/cli/fsm.go index cdf347f0d..78d894f31 100644 --- a/cmd/cli/fsm.go +++ b/cmd/cli/fsm.go @@ -46,6 +46,11 @@ func newRootCmd(config *action.Configuration, stdin io.Reader, stdout io.Writer, newPolicyCmd(stdout, stderr), newSupportCmd(config, stdout, stderr), newUninstallCmd(config, stdin, stdout), + newIngressCmd(config, stdout), + newGatewayCmd(stdout), + newServiceLBCmd(stdout), + newFLBCmd(config, stdout), + newEgressGatewayCmd(config, stdout), ) // Add subcommands related to unmanaged environments diff --git a/cmd/cli/gateway.go b/cmd/cli/gateway.go new file mode 100644 index 000000000..8acac2d15 --- /dev/null +++ b/cmd/cli/gateway.go @@ -0,0 +1,26 @@ +package main + +import ( + "io" + + "github.com/spf13/cobra" +) + +const gatewayDescription = ` +This command consists of multiple subcommands related to managing gateway controller +associated with fsm installations. +` + +func newGatewayCmd(out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "gateway", + Short: "manage fsm gateway", + Aliases: []string{"gw"}, + Long: gatewayDescription, + Args: cobra.NoArgs, + } + cmd.AddCommand(newGatewayEnable(out)) + cmd.AddCommand(newGatewayDisable(out)) + + return cmd +} diff --git a/cmd/cli/gateway_disable.go b/cmd/cli/gateway_disable.go new file mode 100644 index 000000000..8eeb5654d --- /dev/null +++ b/cmd/cli/gateway_disable.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "fmt" + "io" + + gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const gatewayDisableDescription = ` +This command will disable FSM gateway, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type gatewayDisableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + gatewayAPIClient gatewayApiClientset.Interface + meshName string +} + +func newGatewayDisable(out io.Writer) *cobra.Command { + disableCmd := &gatewayDisableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "disable", + Short: "disable fsm gateway", + Long: gatewayDisableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.configClient = configClient + + gatewayAPIClient, err := gatewayApiClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.gatewayAPIClient = gatewayAPIClient + + return disableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&disableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *gatewayDisableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !mc.Spec.GatewayAPI.Enabled { + fmt.Fprintf(cmd.out, "Gateway is disabled already, no action needed\n") + return nil + } + + debug("Deleting FSM Gateway resources ...") + err = deleteGatewayResources(ctx, cmd.gatewayAPIClient) + if err != nil { + return err + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "gatewayAPI.enabled": false, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.GatewayAPI.Enabled = false + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "Gateway is disabled successfully\n") + + return nil +} diff --git a/cmd/cli/gateway_enable.go b/cmd/cli/gateway_enable.go new file mode 100644 index 000000000..5c00586f7 --- /dev/null +++ b/cmd/cli/gateway_enable.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "fmt" + "io" + + nsigClientset "github.com/flomesh-io/fsm/pkg/gen/client/namespacedingress/clientset/versioned" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const gatewayEnableDescription = ` +This command will enable FSM gateway, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type gatewayEnableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + nsigClient nsigClientset.Interface + meshName string + logLevel string +} + +func newGatewayEnable(out io.Writer) *cobra.Command { + enableCmd := &gatewayEnableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "enable", + Short: "enable fsm gateway", + Long: gatewayEnableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.configClient = configClient + + nsigClient, err := nsigClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.nsigClient = nsigClient + + return enableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&enableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + f.StringVar(&enableCmd.logLevel, "log-level", "error", "log level of gateway") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *gatewayEnableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + // check if gateway is enabled, if yes, just return + // TODO: check if GatewayClass is installed and if there's any running gateway instances + if mc.Spec.GatewayAPI.Enabled { + fmt.Fprintf(cmd.out, "Gatweway is enabled already, no action needed\n") + return nil + } + + debug("Deleting FSM Ingress resources ...") + err = deleteIngressResources(ctx, cmd.kubeClient, fsmNamespace, cmd.meshName) + if err != nil { + return err + } + + debug("Deleting FSM NamespacedIngress resources ...") + err = deleteNamespacedIngressResources(ctx, cmd.nsigClient) + if err != nil { + return err + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "ingress.enabled": false, + "ingress.namespaced": false, + "gatewayAPI.enabled": true, + "gatewayAPI.logLevel": cmd.logLevel, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.Ingress.Enabled = false + mc.Spec.Ingress.Namespaced = false + mc.Spec.GatewayAPI.Enabled = true + mc.Spec.GatewayAPI.LogLevel = cmd.logLevel + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "Gateway is enabled successfully\n") + + return nil +} diff --git a/cmd/cli/ingress.go b/cmd/cli/ingress.go new file mode 100644 index 000000000..63b7dc3d8 --- /dev/null +++ b/cmd/cli/ingress.go @@ -0,0 +1,37 @@ +package main + +import ( + "io" + + "helm.sh/helm/v3/pkg/action" + + "github.com/spf13/cobra" +) + +const ingressDescription = ` +This command consists of multiple subcommands related to managing ingress controller +associated with fsm installations. +` + +var ( + ingressManifestFiles = []string{ + "templates/fsm-ingress-class.yaml", + "templates/fsm-ingress-deployment.yaml", + "templates/fsm-ingress-service.yaml", + } +) + +func newIngressCmd(config *action.Configuration, out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "ingress", + Short: "manage fsm ingress", + Aliases: []string{"ing"}, + Long: ingressDescription, + Args: cobra.NoArgs, + } + cmd.AddCommand(newIngressEnable(config, out)) + cmd.AddCommand(newIngressDisable(out)) + cmd.AddCommand(newNamespacedIngressCmd(out)) + + return cmd +} diff --git a/cmd/cli/ingress_disable.go b/cmd/cli/ingress_disable.go new file mode 100644 index 000000000..77a6edc2c --- /dev/null +++ b/cmd/cli/ingress_disable.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "fmt" + "io" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const ingressDisableDescription = ` +This command will disable FSM ingress, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type ingressDisableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + meshName string +} + +func newIngressDisable(out io.Writer) *cobra.Command { + disableCmd := &ingressDisableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "disable", + Short: "disable fsm ingress", + Long: ingressDisableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.configClient = configClient + + return disableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&disableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *ingressDisableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + // check if ingress is enabled, if yes, just return + if !mc.Spec.Ingress.Enabled { + fmt.Fprintf(cmd.out, "Ingress is disabled already, no action needed\n") + return nil + } + + debug("Deleting FSM Ingress resources ...") + err = deleteIngressResources(ctx, cmd.kubeClient, fsmNamespace, cmd.meshName) + if err != nil { + return err + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "ingress.enabled": false, + "ingress.namespaced": false, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.Ingress.Enabled = false + mc.Spec.Ingress.Namespaced = false + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "Ingress is disabled successfully\n") + + return nil +} diff --git a/cmd/cli/ingress_enable.go b/cmd/cli/ingress_enable.go new file mode 100644 index 000000000..82fc742c1 --- /dev/null +++ b/cmd/cli/ingress_enable.go @@ -0,0 +1,283 @@ +package main + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/flomesh-io/fsm/pkg/constants" + + gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + + nsigClientset "github.com/flomesh-io/fsm/pkg/gen/client/namespacedingress/clientset/versioned" + + "helm.sh/helm/v3/pkg/action" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/restmapper" + + configv1alpha3 "github.com/flomesh-io/fsm/pkg/apis/config/v1alpha3" + + "k8s.io/client-go/dynamic" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ingressEnableDescription = ` +This command will enable FSM ingress, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type ingressEnableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + dynamicClient dynamic.Interface + configClient configClientset.Interface + nsigClient nsigClientset.Interface + gatewayAPIClient gatewayApiClientset.Interface + mapper meta.RESTMapper + actionConfig *action.Configuration + meshName string + logLevel string + httpEnabled bool + httpPort int32 + httpNodePort int32 + tlsEnabled bool + mtls bool + tlsPort int32 + tlsNodePort int32 + passthroughEnabled bool + passthroughUpstreamPort int32 + replicas int32 +} + +func (cmd *ingressEnableCmd) GetActionConfig() *action.Configuration { + return cmd.actionConfig +} + +func (cmd *ingressEnableCmd) GetDynamicClient() dynamic.Interface { + return cmd.dynamicClient +} + +func (cmd *ingressEnableCmd) GetRESTMapper() meta.RESTMapper { + return cmd.mapper +} + +func (cmd *ingressEnableCmd) GetMeshName() string { + return cmd.meshName +} + +func newIngressEnable(actionConfig *action.Configuration, out io.Writer) *cobra.Command { + enableCmd := &ingressEnableCmd{ + out: out, + actionConfig: actionConfig, + } + + cmd := &cobra.Command{ + Use: "enable", + Short: "enable fsm ingress", + Long: ingressEnableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.kubeClient = kubeClient + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.dynamicClient = dynamicClient + + gr, err := restmapper.GetAPIGroupResources(kubeClient.Discovery()) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + + mapper := restmapper.NewDiscoveryRESTMapper(gr) + enableCmd.mapper = mapper + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.configClient = configClient + + nsigClient, err := nsigClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.nsigClient = nsigClient + + gatewayAPIClient, err := gatewayApiClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.gatewayAPIClient = gatewayAPIClient + + return enableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&enableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + f.StringVar(&enableCmd.logLevel, "log-level", "error", "log level of ingress") + f.BoolVar(&enableCmd.httpEnabled, "http-enable", true, "enable/disable HTTP ingress") + f.Int32Var(&enableCmd.httpPort, "http-port", 80, "HTTP ingress port") + f.Int32Var(&enableCmd.httpNodePort, "http-node-port", 30508, "HTTP ingress node port") + f.BoolVar(&enableCmd.tlsEnabled, "tls-enable", false, "enable/disable TLS ingress") + f.BoolVar(&enableCmd.mtls, "mtls", false, "enable/disable mTLS for ingress") + f.Int32Var(&enableCmd.tlsPort, "tls-port", 443, "TLS ingress port") + f.Int32Var(&enableCmd.tlsNodePort, "tls-node-port", 30607, "TLS ingress node port") + f.BoolVar(&enableCmd.passthroughEnabled, "passthrough-enable", false, "enable/disable SSL passthrough") + f.Int32Var(&enableCmd.passthroughUpstreamPort, "passthrough-upstream-port", 443, "SSL passthrough upstream port") + f.Int32Var(&enableCmd.replicas, "replicas", 1, "replicas of ingress") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *ingressEnableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + // check if ingress is enabled, if yes, just return + // TODO: check if ingress controller is installed and running??? + if mc.Spec.Ingress.Enabled { + fmt.Fprintf(cmd.out, "Ingress is enabled, no action needed\n") + return nil + } + + debug("Deleting FSM NamespacedIngress resources ...") + err = deleteNamespacedIngressResources(ctx, cmd.nsigClient) + if err != nil { + return err + } + + debug("Deleting FSM Gateway resources ...") + err = deleteGatewayResources(ctx, cmd.gatewayAPIClient) + if err != nil { + return err + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "ingress.enabled": true, + "ingress.namespaced": false, + "ingress.logLevel": cmd.logLevel, + "ingress.http.enabled": cmd.httpEnabled, + "ingress.http.bind": cmd.httpPort, + "ingress.http.nodePort": cmd.httpNodePort, + "ingress.tls.enabled": cmd.tlsEnabled, + "ingress.tls.bind": cmd.tlsPort, + "ingress.tls.nodePort": cmd.tlsNodePort, + "ingress.tls.mTLS": cmd.mtls, + "ingress.tls.sslPassthrough.enabled": cmd.passthroughEnabled, + "ingress.tls.sslPassthrough.upstreamPort": cmd.passthroughUpstreamPort, + "gatewayAPI.enabled": false, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.Ingress.Enabled = true + mc.Spec.Ingress.Namespaced = false + mc.Spec.Ingress.LogLevel = cmd.logLevel + mc.Spec.Ingress.HTTP.Enabled = cmd.httpEnabled + mc.Spec.Ingress.HTTP.Bind = cmd.httpPort + mc.Spec.Ingress.HTTP.NodePort = cmd.httpNodePort + mc.Spec.Ingress.TLS.Enabled = cmd.tlsEnabled + mc.Spec.Ingress.TLS.Bind = cmd.tlsPort + mc.Spec.Ingress.TLS.NodePort = cmd.tlsNodePort + mc.Spec.Ingress.TLS.MTLS = cmd.mtls + mc.Spec.Ingress.TLS.SSLPassthrough.Enabled = cmd.passthroughEnabled + mc.Spec.Ingress.TLS.SSLPassthrough.UpstreamPort = cmd.passthroughUpstreamPort + mc.Spec.GatewayAPI.Enabled = false + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + debug("Restarting fsm-controller ...") + // Rollout restart fsm-controller + // patch the deployment spec template triggers the action of rollout restart like with kubectl + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + if err := installManifests(cmd, mc, fsmNamespace, kubeVersion119, ingressManifestFiles...); err != nil { + return err + } + + time.Sleep(3 * time.Second) + + deployment, err := cmd.kubeClient.AppsV1(). + Deployments(fsmNamespace). + Get(ctx, constants.FSMIngressName, metav1.GetOptions{}) + if err != nil { + return err + } + + if err := waitForDeploymentReady(ctx, cmd.kubeClient, deployment, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "Ingress is enabled successfully\n") + + return nil +} + +func (cmd *ingressEnableCmd) ResolveValues(mc *configv1alpha3.MeshConfig) (map[string]interface{}, error) { + finalValues := map[string]interface{}{} + + valuesConfig := []string{ + fmt.Sprintf("fsm.fsmIngress.enabled=%t", true), + fmt.Sprintf("fsm.fsmIngress.namespaced=%t", false), + fmt.Sprintf("fsm.fsmIngress.logLevel=%s", cmd.logLevel), + fmt.Sprintf("fsm.fsmIngress.http.enabled=%t", cmd.httpEnabled), + fmt.Sprintf("fsm.fsmIngress.http.port=%d", cmd.httpPort), + fmt.Sprintf("fsm.fsmIngress.http.nodePort=%d", cmd.httpNodePort), + fmt.Sprintf("fsm.fsmIngress.tls.enabled=%t", cmd.tlsEnabled), + fmt.Sprintf("fsm.fsmIngress.tls.port=%d", cmd.tlsPort), + fmt.Sprintf("fsm.fsmIngress.tls.nodePort=%d", cmd.tlsNodePort), + fmt.Sprintf("fsm.fsmIngress.tls.mTLS=%t", cmd.mtls), + fmt.Sprintf("fsm.fsmIngress.tls.sslPassthrough.enabled=%t", cmd.passthroughEnabled), + fmt.Sprintf("fsm.fsmIngress.tls.sslPassthrough.upstreamPort=%d", cmd.passthroughUpstreamPort), + fmt.Sprintf("fsm.fsmIngress.replicaCount=%d", cmd.replicas), + fmt.Sprintf("fsm.fsmGateway.enabled=%t", false), + fmt.Sprintf("fsm.fsmNamespace=%s", mc.GetNamespace()), + fmt.Sprintf("fsm.meshName=%s", cmd.meshName), + fmt.Sprintf("fsm.image.registry=%s", mc.Spec.Image.Registry), + fmt.Sprintf("fsm.image.pullPolicy=%s", mc.Spec.Image.PullPolicy), + fmt.Sprintf("fsm.image.tag=%s", mc.Spec.Image.Tag), + fmt.Sprintf("fsm.curlImage=%s", mc.Spec.Misc.CurlImage), + } + + if err := parseVal(valuesConfig, finalValues); err != nil { + return nil, err + } + + return finalValues, nil +} diff --git a/cmd/cli/ingress_namespaced.go b/cmd/cli/ingress_namespaced.go new file mode 100644 index 000000000..281d6a5a5 --- /dev/null +++ b/cmd/cli/ingress_namespaced.go @@ -0,0 +1,21 @@ +package main + +import ( + "io" + + "github.com/spf13/cobra" +) + +func newNamespacedIngressCmd(out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "namespaced", + Short: "manage fsm NamespacedIngress", + Aliases: []string{"nsig"}, + Long: ingressDescription, + Args: cobra.NoArgs, + } + cmd.AddCommand(newNamespacedIngressEnableCmd(out)) + cmd.AddCommand(newNamespacedIngressDisableCmd(out)) + + return cmd +} diff --git a/cmd/cli/ingress_namespaced_disable.go b/cmd/cli/ingress_namespaced_disable.go new file mode 100644 index 000000000..5fe54e247 --- /dev/null +++ b/cmd/cli/ingress_namespaced_disable.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "fmt" + "io" + + nsigClientset "github.com/flomesh-io/fsm/pkg/gen/client/namespacedingress/clientset/versioned" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const namespacedIngressDisableDescription = ` +This command will disable FSM NamespacedIngress, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type namespacedIngressDisableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + nsigClient nsigClientset.Interface + meshName string +} + +func newNamespacedIngressDisableCmd(out io.Writer) *cobra.Command { + disableCmd := &namespacedIngressDisableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "disable", + Short: "disable fsm NamespacedIngress", + Long: namespacedIngressDisableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.configClient = configClient + + nsigClient, err := nsigClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.nsigClient = nsigClient + + return disableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&disableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *namespacedIngressDisableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !mc.Spec.Ingress.Enabled && !mc.Spec.Ingress.Namespaced { + fmt.Fprintf(cmd.out, "NamespacedIngress is disabled already, no action needed\n") + return nil + } + + debug("Deleting FSM NamespacedIngress resources ...") + err = deleteNamespacedIngressResources(ctx, cmd.nsigClient) + if err != nil { + return err + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "ingress.enabled": false, + "ingress.namespaced": false, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.Ingress.Enabled = false + mc.Spec.Ingress.Namespaced = false + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "NamespacedIngress is disabled successfully\n") + + return nil +} diff --git a/cmd/cli/ingress_namespaced_enable.go b/cmd/cli/ingress_namespaced_enable.go new file mode 100644 index 000000000..3b0eb6a6b --- /dev/null +++ b/cmd/cli/ingress_namespaced_enable.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "fmt" + "io" + + gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" + + "github.com/spf13/cobra" +) + +const namespacedIngressEnableDescription = ` +This command will enable FSM NamespacedIngress, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type namespacedIngressEnableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + gatewayAPIClient gatewayApiClientset.Interface + meshName string +} + +func newNamespacedIngressEnableCmd(out io.Writer) *cobra.Command { + enableCmd := &namespacedIngressEnableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "enable", + Short: "enable fsm NamespacedIngress", + Long: namespacedIngressEnableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.configClient = configClient + + gatewayAPIClient, err := gatewayApiClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.gatewayAPIClient = gatewayAPIClient + + return enableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&enableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *namespacedIngressEnableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if mc.Spec.Ingress.Enabled && mc.Spec.Ingress.Namespaced { + fmt.Fprintf(cmd.out, "NamespacedIngress is enabled already, no action needed\n") + return nil + } + + debug("Deleting FSM Ingress resources ...") + err = deleteIngressResources(ctx, cmd.kubeClient, fsmNamespace, cmd.meshName) + if err != nil { + return err + } + + debug("Deleting FSM Gateway resources ...") + err = deleteGatewayResources(ctx, cmd.gatewayAPIClient) + if err != nil { + return err + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "ingress.enabled": true, + "ingress.namespaced": true, + "gatewayAPI.enabled": false, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.Ingress.Enabled = true + mc.Spec.Ingress.Namespaced = true + mc.Spec.GatewayAPI.Enabled = false + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "NamespacedIngress is enabled successfully\n") + + return nil +} diff --git a/cmd/cli/install.go b/cmd/cli/install.go index 084001819..933a98dad 100644 --- a/cmd/cli/install.go +++ b/cmd/cli/install.go @@ -82,6 +82,7 @@ type installCmd struct { disableSpinner bool valueFiles []string // -f/--values + dryRun bool } func newInstallCmd(config *helm.Configuration, out io.Writer) *cobra.Command { @@ -116,6 +117,7 @@ func newInstallCmd(config *helm.Configuration, out io.Writer) *cobra.Command { f.StringArrayVar(&inst.setOptions, "set", nil, "Set arbitrary chart values (can specify multiple or separate values with commas: key1=val1,key2=val2)") f.BoolVar(&inst.atomic, "atomic", false, "Automatically clean up resources if installation fails") f.StringSliceVarP(&inst.valueFiles, "values", "f", []string{}, "Specify values in a YAML file (can specify multiple)") + f.BoolVar(&inst.dryRun, "dry-run", false, "Simulate an install and output rendered manifests") return cmd } @@ -149,6 +151,18 @@ func (i *installCmd) run(config *helm.Configuration) error { installClient.Wait = true installClient.Atomic = i.atomic installClient.Timeout = i.timeout + installClient.DryRun = i.dryRun + + if i.dryRun { + rel, err := installClient.Run(i.chartRequested, values) + if err != nil { + return fmt.Errorf("error rendering templates: %s", err) + } + + fmt.Fprintf(i.out, "%s", rel.Manifest) + + return nil + } debug("Beginning FSM installation") if i.disableSpinner || settings.Verbose() { diff --git a/cmd/cli/namespacedingress.go b/cmd/cli/namespacedingress.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/cmd/cli/namespacedingress.go @@ -0,0 +1 @@ +package main diff --git a/cmd/cli/servicelb.go b/cmd/cli/servicelb.go new file mode 100644 index 000000000..df694a4d0 --- /dev/null +++ b/cmd/cli/servicelb.go @@ -0,0 +1,26 @@ +package main + +import ( + "io" + + "github.com/spf13/cobra" +) + +const serviceLBDescription = ` +This command consists of multiple subcommands related to managing service-lb controller +associated with fsm installations. +` + +func newServiceLBCmd(out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "servicelb", + Short: "manage fsm service-lb", + Aliases: []string{"slb"}, + Long: serviceLBDescription, + Args: cobra.NoArgs, + } + cmd.AddCommand(newServiceLBEnableCmd(out)) + cmd.AddCommand(newServiceLBDisableCmd(out)) + + return cmd +} diff --git a/cmd/cli/servicelb_disable.go b/cmd/cli/servicelb_disable.go new file mode 100644 index 000000000..9a75cad37 --- /dev/null +++ b/cmd/cli/servicelb_disable.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const serviceLBDisableDescription = ` +This command will disable FSM service-lb, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type serviceLBDisableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + meshName string +} + +func newServiceLBDisableCmd(out io.Writer) *cobra.Command { + disableCmd := &serviceLBDisableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "disable", + Short: "disable fsm service-lb", + Long: serviceLBDisableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + disableCmd.configClient = configClient + + return disableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&disableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *serviceLBDisableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if !mc.Spec.ServiceLB.Enabled { + fmt.Fprintf(cmd.out, "service-lb is disabled already, no action needed\n") + return nil + } + + if err := updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "serviceLB.enabled": false, + }); err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.ServiceLB.Enabled = false + if _, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}); err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + debug("Deleting FSM service-lb resources ...") + if err := deleteServiceLBResources(ctx, cmd.kubeClient, fsmNamespace, cmd.meshName); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "service-lb is disabled successfully\n") + + return nil +} diff --git a/cmd/cli/servicelb_enable.go b/cmd/cli/servicelb_enable.go new file mode 100644 index 000000000..f5f815e53 --- /dev/null +++ b/cmd/cli/servicelb_enable.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configClientset "github.com/flomesh-io/fsm/pkg/gen/client/config/clientset/versioned" +) + +const serviceLBEnableDescription = ` +This command will enable FSM service-lb, make sure --mesh-name and --fsm-namespace matches +the release name and namespace of installed FSM, otherwise it doesn't work. +` + +type serviceLBEnableCmd struct { + out io.Writer + kubeClient kubernetes.Interface + configClient configClientset.Interface + meshName string +} + +func newServiceLBEnableCmd(out io.Writer) *cobra.Command { + enableCmd := &serviceLBEnableCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "enable", + Short: "enable fsm service-lb", + Long: serviceLBEnableDescription, + Args: cobra.ExactArgs(0), + RunE: func(_ *cobra.Command, args []string) error { + config, err := settings.RESTClientGetter().ToRESTConfig() + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.kubeClient = kubeClient + + configClient, err := configClientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not access Kubernetes cluster, check kubeconfig: %w", err) + } + enableCmd.configClient = configClient + + return enableCmd.run() + }, + } + + f := cmd.Flags() + f.StringVar(&enableCmd.meshName, "mesh-name", defaultMeshName, "name for the control plane instance") + //utilruntime.Must(cmd.MarkFlagRequired("mesh-name")) + + return cmd +} + +func (cmd *serviceLBEnableCmd) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fsmNamespace := settings.Namespace() + + debug("Getting mesh config ...") + // get mesh config + mc, err := cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Get(ctx, defaultFsmMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + if mc.Spec.ServiceLB.Enabled { + fmt.Fprintf(cmd.out, "service-lb is enabled already, no action needed\n") + return nil + } + + err = updatePresetMeshConfigMap(ctx, cmd.kubeClient, fsmNamespace, map[string]interface{}{ + "serviceLB.enabled": true, + }) + if err != nil { + return err + } + + debug("Updating mesh config ...") + // update mesh config, fsm-mesh-config + mc.Spec.ServiceLB.Enabled = true + _, err = cmd.configClient.ConfigV1alpha3().MeshConfigs(fsmNamespace).Update(ctx, mc, metav1.UpdateOptions{}) + if err != nil { + return err + } + + if err := restartFSMController(ctx, cmd.kubeClient, fsmNamespace, cmd.out); err != nil { + return err + } + + fmt.Fprintf(cmd.out, "service-lb is enabled successfully\n") + + return nil +} diff --git a/cmd/cli/shared.go b/cmd/cli/shared.go new file mode 100644 index 000000000..7d21373f8 --- /dev/null +++ b/cmd/cli/shared.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + + "helm.sh/helm/v3/pkg/chartutil" +) + +const ( + presetMeshConfigName = "preset-mesh-config" + presetMeshConfigJSONKey = "preset-mesh-config.json" +) + +var ( + kubeVersion119 = &chartutil.KubeVersion{ + Version: fmt.Sprintf("v%s.%s.0", "1", "19"), + Major: "1", + Minor: "19", + } + + //kubeVersion121 = &chartutil.KubeVersion{ + // Version: fmt.Sprintf("v%s.%s.0", "1", "21"), + // Major: "1", + // Minor: "21", + //} +) diff --git a/cmd/cli/util.go b/cmd/cli/util.go index f4eec6b8b..445c90194 100644 --- a/cmd/cli/util.go +++ b/cmd/cli/util.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "bytes" "context" "encoding/json" "fmt" @@ -9,6 +10,38 @@ import ( "net/http" "sort" "strings" + "time" + + deploymentutil "k8s.io/kubectl/pkg/util/deployment" + + "k8s.io/kubectl/pkg/util/interrupt" + + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" + + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + + "github.com/flomesh-io/fsm/pkg/helm" + + configv1alpha3 "github.com/flomesh-io/fsm/pkg/apis/config/v1alpha3" + + "helm.sh/helm/v3/pkg/action" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/dynamic" + + gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + + nsigClientset "github.com/flomesh-io/fsm/pkg/gen/client/namespacedingress/clientset/versioned" + + "k8s.io/apimachinery/pkg/api/errors" + + "github.com/tidwall/sjson" + + "k8s.io/apimachinery/pkg/types" mapset "github.com/deckarep/golang-set" appsv1 "k8s.io/api/apps/v1" @@ -17,11 +50,20 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + watchtools "k8s.io/client-go/tools/watch" "github.com/flomesh-io/fsm/pkg/constants" "github.com/flomesh-io/fsm/pkg/k8s" ) +type ManifestClient interface { + GetActionConfig() *action.Configuration + GetDynamicClient() dynamic.Interface + GetRESTMapper() meta.RESTMapper + GetMeshName() string + ResolveValues(mc *configv1alpha3.MeshConfig) (map[string]interface{}, error) +} + // confirm displays a prompt `s` to the user and returns a bool indicating yes / no // If the lowercased, trimmed input begins with anything other than 'y', it returns false // It accepts an int `tries` representing the number of attempts before returning false @@ -262,3 +304,388 @@ func annotateErrorMessageWithActionableMessage(actionableMessage string, errMsgF return fmt.Errorf(errMsgFormat+actionableMessage, args...) } + +//lint:ignore U1000 ignore unused +func restartFSMController(ctx context.Context, kubeClient kubernetes.Interface, fsmNamespace string, out io.Writer) error { + debug("Restarting fsm-controller ...") + // Rollout restart fsm-controller + // patch the deployment spec template triggers the action of rollout restart like with kubectl + patch := fmt.Sprintf( + `{"spec": {"template":{"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": "%s"}}}}}`, + time.Now().Format("20060102-150405.0000"), + ) + + deployment, err := kubeClient.AppsV1(). + Deployments(fsmNamespace). + Patch(ctx, constants.FSMControllerName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}) + if err != nil { + return err + } + + if err := waitForDeploymentReady(ctx, kubeClient, deployment, out); err != nil { + return err + } + + return nil +} + +func waitForDeploymentReady(ctx context.Context, kubeClient kubernetes.Interface, deployment *appsv1.Deployment, out io.Writer) error { + timeout := 5 * time.Minute + + fieldSelector := fields.OneTermEqualSelector("metadata.name", deployment.Name).String() + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = fieldSelector + return kubeClient.AppsV1().Deployments(deployment.Namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = fieldSelector + return kubeClient.AppsV1().Deployments(deployment.Namespace).Watch(context.TODO(), options) + }, + } + + // if the rollout isn't done yet, keep watching deployment status + ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout) + intr := interrupt.New(nil, cancel) + if err := intr.Run(func() error { + _, err := watchtools.UntilWithSync(ctx, lw, &appsv1.Deployment{}, nil, func(e watch.Event) (bool, error) { + switch t := e.Type; t { + case watch.Added, watch.Modified: + status, done, err := deploymentStatus(e.Object.(*appsv1.Deployment)) + if err != nil { + return false, err + } + fmt.Fprintf(out, "%s", status) + // Quit waiting if the rollout is done + if done { + return true, nil + } + + return false, nil + case watch.Deleted: + // We need to abort to avoid cases of recreation and not to silently watch the wrong (new) object + return true, fmt.Errorf("object has been deleted") + default: + return true, fmt.Errorf("internal error: unexpected event %#v", e) + } + }) + return err + }); err != nil { + return err + } + + return nil +} + +func deploymentStatus(deployment *appsv1.Deployment) (string, bool, error) { + if deployment.Generation <= deployment.Status.ObservedGeneration { + cond := deploymentutil.GetDeploymentCondition(deployment.Status, appsv1.DeploymentProgressing) + + if cond != nil && cond.Reason == deploymentutil.TimedOutReason { + return "", false, fmt.Errorf("deployment %q exceeded its progress deadline", deployment.Name) + } + if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d out of %d new replicas have been updated...\n", deployment.Name, deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas), false, nil + } + if deployment.Status.Replicas > deployment.Status.UpdatedReplicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d old replicas are pending termination...\n", deployment.Name, deployment.Status.Replicas-deployment.Status.UpdatedReplicas), false, nil + } + if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { + return fmt.Sprintf("Waiting for deployment %q rollout to finish: %d of %d updated replicas are available...\n", deployment.Name, deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas), false, nil + } + + return fmt.Sprintf("deployment %q successfully rolled out\n", deployment.Name), true, nil + } + + return fmt.Sprintf("Waiting for deployment %q spec update to be observed...\n", deployment.Name), false, nil +} + +//lint:ignore U1000 ignore unused +func waitForPodsRunningReady(kubeClient kubernetes.Interface, ns string, nExpectedRunningPods int, labelSelector *metav1.LabelSelector) error { + timeout := 5 * time.Minute + debug("Wait up to %v for %d pods ready in ns [%s]...", timeout, nExpectedRunningPods, ns) + + listOpts := metav1.ListOptions{ + FieldSelector: "status.phase=Running", + } + + if labelSelector != nil { + labelMap, _ := metav1.LabelSelectorAsMap(labelSelector) + listOpts.LabelSelector = labels.SelectorFromSet(labelMap).String() + } + + for start := time.Now(); time.Since(start) < timeout; time.Sleep(2 * time.Second) { + pods, err := kubeClient.CoreV1().Pods(ns).List(context.TODO(), listOpts) + + if err != nil { + return fmt.Errorf("failed to list pods") + } + + if len(pods.Items) < nExpectedRunningPods { + time.Sleep(time.Second) + continue + } + + nReadyPods := 0 + for _, pod := range pods.Items { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + nReadyPods++ + if nReadyPods == nExpectedRunningPods { + debug("Finished waiting for NS [%s].", ns) + return nil + } + } + } + } + time.Sleep(time.Second) + } + + pods, err := kubeClient.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list pods") + } + debug("Pod Statuses in namespace", ns) + for _, pod := range pods.Items { + status, _ := json.MarshalIndent(pod.Status, "", " ") + debug("Pod %s:\n%s", pod.Name, status) + } + + return fmt.Errorf("not all pods were Running & Ready in NS %s after %v", ns, timeout) +} + +func updatePresetMeshConfigMap(ctx context.Context, kubeClient kubernetes.Interface, fsmNamespace string, values map[string]interface{}) error { + debug("Getting configmap preset-mesh-config ...") + // get configmap preset-mesh-config + cm, err := kubeClient.CoreV1().ConfigMaps(fsmNamespace).Get(ctx, presetMeshConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + debug("Updating configmap preset-mesh-config ...") + // update content data of preset-mesh-config.json + presetMeshConfigJSON := cm.Data[presetMeshConfigJSONKey] + for path, value := range values { + presetMeshConfigJSON, err = sjson.Set(presetMeshConfigJSON, path, value) + if err != nil { + return err + } + } + + // update configmap preset-mesh-config + cm.Data[presetMeshConfigJSONKey] = presetMeshConfigJSON + if _, err := kubeClient.CoreV1().ConfigMaps(fsmNamespace).Update(ctx, cm, metav1.UpdateOptions{}); err != nil { + return err + } + + return nil +} + +func deleteIngressResources(ctx context.Context, kubeClient kubernetes.Interface, fsmNamespace, meshName string) error { + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + constants.AppLabel: constants.FSMIngressName, + "meshName": meshName, + "ingress.flomesh.io/namespaced": "false", + }, + } + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + } + + serviceList, err := kubeClient.CoreV1().Services(fsmNamespace).List(ctx, listOptions) + if err != nil { + return err + } + for _, service := range serviceList.Items { + if err := kubeClient.CoreV1().Services(fsmNamespace).Delete(ctx, service.Name, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + } + + deploymentList, err := kubeClient.AppsV1().Deployments(fsmNamespace).List(ctx, listOptions) + if err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if err := kubeClient.AppsV1().Deployments(fsmNamespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + } + + if err := kubeClient.NetworkingV1().IngressClasses().Delete(ctx, constants.IngressPipyClass, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + + return nil +} + +func deleteNamespacedIngressResources(ctx context.Context, nsigClient nsigClientset.Interface) error { + nsigList, err := nsigClient.FlomeshV1alpha1().NamespacedIngresses(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + + for _, nsig := range nsigList.Items { + if err := nsigClient.FlomeshV1alpha1().NamespacedIngresses(nsig.GetNamespace()).Delete(ctx, nsig.GetName(), metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + } + + return nil +} + +func deleteGatewayResources(ctx context.Context, gatewayAPIClient gatewayApiClientset.Interface) error { + // delete gateways + debug("Deleting gateways ...") + gatewayList, err := gatewayAPIClient.GatewayV1beta1().Gateways(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + + for _, gateway := range gatewayList.Items { + if err := gatewayAPIClient.GatewayV1beta1().Gateways(gateway.GetNamespace()).Delete(ctx, gateway.GetName(), metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + } + + // delete gatewayclasses + debug("Deleting gatewayclasses ...") + gatewayClassList, err := gatewayAPIClient.GatewayV1beta1().GatewayClasses().List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + + for _, gatewayClass := range gatewayClassList.Items { + if err := gatewayAPIClient.GatewayV1beta1().GatewayClasses().Delete(ctx, gatewayClass.GetName(), metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + } + + return nil +} + +func deleteEgressGatewayResources(ctx context.Context, kubeClient kubernetes.Interface, fsmNamespace, meshName string) error { + if err := kubeClient.CoreV1().Services(fsmNamespace).Delete(ctx, constants.FSMEgressGatewayName, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + + if err := kubeClient.AppsV1().Deployments(fsmNamespace).Delete(ctx, constants.FSMEgressGatewayName, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + + if err := kubeClient.CoreV1().ConfigMaps(fsmNamespace).Delete(ctx, "fsm-egress-gateway-pjs", metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + + return nil +} + +func deleteServiceLBResources(ctx context.Context, kubeClient kubernetes.Interface, fsmNamespace, meshName string) error { + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + constants.AppLabel: constants.FSMServiceLBName, + "meshName": meshName, + }, + } + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + } + + daemonSetList, err := kubeClient.AppsV1().DaemonSets(fsmNamespace).List(ctx, listOptions) + if err != nil { + return err + } + for _, daemonSet := range daemonSetList.Items { + if err := kubeClient.AppsV1().DaemonSets(fsmNamespace).Delete(ctx, daemonSet.Name, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + } + + return nil +} + +func deleteFLBResources(ctx context.Context, kubeClient kubernetes.Interface) error { + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + constants.FLBSecretLabel: "true", + }, + } + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + } + + secretList, err := kubeClient.CoreV1().Secrets(corev1.NamespaceAll).List(ctx, listOptions) + if err != nil { + return err + } + for _, secret := range secretList.Items { + if err := kubeClient.AppsV1().DaemonSets(secret.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}); err != nil { + if !errors.IsNotFound(err) { + return err + } + } + } + + return nil +} + +func installManifests(cmd ManifestClient, mc *configv1alpha3.MeshConfig, fsmNamespace string, kubeVersion *chartutil.KubeVersion, manifestFiles ...string) error { + debug("Loading fsm helm chart ...") + // load fsm helm chart + chart, err := loader.LoadArchive(bytes.NewReader(chartTGZSource)) + if err != nil { + return err + } + + debug("Resolving values ...") + // resolve values + values, err := cmd.ResolveValues(mc) + if err != nil { + return err + } + + debug("Creating helm template client ...") + // create a helm template client + templateClient := helm.TemplateClient( + cmd.GetActionConfig(), + cmd.GetMeshName(), + fsmNamespace, + kubeVersion, + ) + templateClient.Replace = true + + debug("Rendering helm template ...") + // render entire fsm helm template + rel, err := templateClient.Run(chart, values) + if err != nil { + return err + } + + debug("Apply manifests ...") + // filter out unneeded manifests, only keep interested manifests, then do a kubectl-apply like action for each manifest + if err := helm.ApplyYAMLs(cmd.GetDynamicClient(), cmd.GetRESTMapper(), rel.Manifest, helm.ApplyManifest, manifestFiles...); err != nil { + return err + } + return nil +} diff --git a/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml b/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml index 44ebdc64e..cda7693af 100644 --- a/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml +++ b/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml @@ -1516,6 +1516,16 @@ spec: - logLevel - namespaced type: object + misc: + description: Misc defines the configurations of misc info + properties: + curlImage: + default: curlimages/curl + description: CurlImage defines the image of curl. + type: string + required: + - curlImage + type: object observability: description: Observalility defines the observability configurations for a mesh instance. @@ -1714,7 +1724,7 @@ spec: description: Enabled defines if service lb is enabled. type: boolean image: - default: mirrored-klipper-lb:v0.3.5 + default: flomesh/mirrored-klipper-lb:v0.3.5 description: Image defines the service lb image. type: string required: @@ -2005,6 +2015,7 @@ spec: type: object required: - image + - misc type: object type: object served: true diff --git a/pkg/apis/config/v1alpha3/mesh_config.go b/pkg/apis/config/v1alpha3/mesh_config.go index 0cdebf0f5..894a4b8b3 100644 --- a/pkg/apis/config/v1alpha3/mesh_config.go +++ b/pkg/apis/config/v1alpha3/mesh_config.go @@ -63,8 +63,14 @@ type MeshConfigSpec struct { // FLB defines the configurations of FLB features. FLB FLBSpec `json:"flb,omitempty"` + // EgressGateway defines the configurations of EgressGateway features. + EgressGateway EgressGatewaySpec `json:"egressGateway,omitempty"` + // Image defines the configurations of Image info Image ImageSpec `json:"image"` + + // Misc defines the configurations of misc info + Misc MiscSpec `json:"misc"` } // LocalProxyMode is a type alias representing the way the sidecar proxies to the main application @@ -546,7 +552,7 @@ type ServiceLBSpec struct { // Enabled defines if service lb is enabled. Enabled bool `json:"enabled"` - // +kubebuilder:default="mirrored-klipper-lb:v0.3.5" + // +kubebuilder:default="flomesh/mirrored-klipper-lb:v0.3.5" // Image defines the service lb image. Image string `json:"image"` } @@ -566,6 +572,39 @@ type FLBSpec struct { SecretName string `json:"secretName"` } +// EgressGatewaySpec is the type to represent egress gateway. +type EgressGatewaySpec struct { + // +kubebuilder:default=false + // Enabled defines if flb is enabled. + Enabled bool `json:"enabled"` + + // +kubebuilder:default=info + // +kubebuilder:validation:Enum=trace;debug;info;warn;error;fatal;panic;disabled + // LogLevel defines the log level of gateway api. + LogLevel string `json:"logLevel"` + + // +kubebuilder:default=http2tunnel + // +kubebuilder:validation:Enum=http2tunnel;sock5 + // Mode defines the mode of egress gateway. + Mode string `json:"mode"` + + // +kubebuilder:default=1080 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // Port defines the port of egress gateway. + Port *int32 `json:"port,omitempty"` + + // +kubebuilder:default=6060 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // AdminPort defines the admin port of egress gateway. + AdminPort *int32 `json:"adminPort,omitempty"` + + // +kubebuilder:default=1 + // Replicas defines the replicas of egress gateway. + Replicas *int32 `json:"replicas,omitempty"` +} + // ImageSpec is the type to represent image. type ImageSpec struct { // +kubebuilder:default=flomesh @@ -580,3 +619,14 @@ type ImageSpec struct { // PullPolicy defines the pull policy of docker image. PullPolicy corev1.PullPolicy `json:"pullPolicy"` } + +// MiscSpec is the type to represent misc configs. +type MiscSpec struct { + // +kubebuilder:default="curlimages/curl" + // CurlImage defines the image of curl. + CurlImage string `json:"curlImage"` + + // +kubebuilder:default="flomesh/pipy-repo:0.90.2-41" + // RepoServerImage defines the image of repo server. + RepoServerImage string `json:"repoServerImage"` +} diff --git a/pkg/apis/config/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha3/zz_generated.deepcopy.go index c867b4ed1..034158b18 100644 --- a/pkg/apis/config/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha3/zz_generated.deepcopy.go @@ -96,6 +96,37 @@ func (in *ClusterSetSpec) DeepCopy() *ClusterSetSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressGatewaySpec) DeepCopyInto(out *EgressGatewaySpec) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } + if in.AdminPort != nil { + in, out := &in.AdminPort, &out.AdminPort + *out = new(int32) + **out = **in + } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressGatewaySpec. +func (in *EgressGatewaySpec) DeepCopy() *EgressGatewaySpec { + if in == nil { + return nil + } + out := new(EgressGatewaySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalAuthzSpec) DeepCopyInto(out *ExternalAuthzSpec) { *out = *in @@ -331,7 +362,9 @@ func (in *MeshConfigSpec) DeepCopyInto(out *MeshConfigSpec) { out.GatewayAPI = in.GatewayAPI out.ServiceLB = in.ServiceLB out.FLB = in.FLB + in.EgressGateway.DeepCopyInto(&out.EgressGateway) out.Image = in.Image + out.Misc = in.Misc return } @@ -439,6 +472,22 @@ func (in *MeshRootCertificateStatus) DeepCopy() *MeshRootCertificateStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MiscSpec) DeepCopyInto(out *MiscSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MiscSpec. +func (in *MiscSpec) DeepCopy() *MiscSpec { + if in == nil { + return nil + } + out := new(MiscSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObservabilitySpec) DeepCopyInto(out *ObservabilitySpec) { *out = *in diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 66b5a2e38..2b49e4c47 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -99,6 +99,12 @@ const ( // FSMGatewayName is the name of the FSM Gateway. FSMGatewayName = "fsm-gateway" + // FSMEgressGatewayName is the name of the FSM Egress Gateway. + FSMEgressGatewayName = "fsm-egress-gateway" + + // FSMServiceLBName is the name of the FSM ServiceLB. + FSMServiceLBName = "fsm-servicelb" + // ProxyServerPort is the port on which the Pipy Repo Service (ADS) listens for new connections from sidecar proxies ProxyServerPort = 6060 diff --git a/pkg/controllers/gateway/v1beta1/gateway_controller.go b/pkg/controllers/gateway/v1beta1/gateway_controller.go index 467df388c..22492f2ef 100644 --- a/pkg/controllers/gateway/v1beta1/gateway_controller.go +++ b/pkg/controllers/gateway/v1beta1/gateway_controller.go @@ -627,13 +627,18 @@ func (r *gatewayReconciler) updateConfig(_ *gwv1beta1.Gateway, _ configurator.Co } func (r *gatewayReconciler) deployGateway(gw *gwv1beta1.Gateway, mc configurator.Configurator) (ctrl.Result, error) { - releaseName := fmt.Sprintf("fsm-gateway-%s", gw.Namespace) - kubeVersion := &chartutil.KubeVersion{ - Version: fmt.Sprintf("v%s.%s.0", "1", "21"), - Major: "1", - Minor: "21", - } - if ctrlResult, err := helm.RenderChart(releaseName, gw, chartSource, mc, r.fctx.Client, r.fctx.Scheme, kubeVersion, r.resolveValues); err != nil { + actionConfig := helm.ActionConfig(gw.Namespace, log.Debug().Msgf) + templateClient := helm.TemplateClient( + actionConfig, + fmt.Sprintf("fsm-gateway-%s", gw.Namespace), + gw.Namespace, + &chartutil.KubeVersion{ + Version: fmt.Sprintf("v%s.%s.0", "1", "21"), + Major: "1", + Minor: "21", + }, + ) + if ctrlResult, err := helm.RenderChart(templateClient, gw, chartSource, mc, r.fctx.Client, r.fctx.Scheme, r.resolveValues); err != nil { defer r.recorder.Eventf(gw, corev1.EventTypeWarning, "Deploy", "Failed to deploy gateway: %s", err) return ctrlResult, err } diff --git a/pkg/controllers/namespacedingress/v1alpha1/namespacedingress_controller.go b/pkg/controllers/namespacedingress/v1alpha1/namespacedingress_controller.go index 655d47ee8..562f295bd 100644 --- a/pkg/controllers/namespacedingress/v1alpha1/namespacedingress_controller.go +++ b/pkg/controllers/namespacedingress/v1alpha1/namespacedingress_controller.go @@ -123,13 +123,18 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrlResult, err } - releaseName := fmt.Sprintf("namespaced-ingress-%s", nsig.Namespace) - kubeVersion := &chartutil.KubeVersion{ - Version: fmt.Sprintf("v%s.%s.0", "1", "19"), - Major: "1", - Minor: "19", - } - if ctrlResult, err = helm.RenderChart(releaseName, nsig, chartSource, mc, r.fctx.Client, r.fctx.Scheme, kubeVersion, r.resolveValues); err != nil { + actionConfig := helm.ActionConfig(nsig.Namespace, log.Debug().Msgf) + templateClient := helm.TemplateClient( + actionConfig, + fmt.Sprintf("namespaced-ingress-%s", nsig.Namespace), + nsig.Namespace, + &chartutil.KubeVersion{ + Version: fmt.Sprintf("v%s.%s.0", "1", "19"), + Major: "1", + Minor: "19", + }, + ) + if ctrlResult, err = helm.RenderChart(templateClient, nsig, chartSource, mc, r.fctx.Client, r.fctx.Scheme, r.resolveValues); err != nil { return ctrlResult, err } diff --git a/pkg/controllers/servicelb/service_controller.go b/pkg/controllers/servicelb/service_controller.go index 3186da7fb..3edea8a0a 100644 --- a/pkg/controllers/servicelb/service_controller.go +++ b/pkg/controllers/servicelb/service_controller.go @@ -31,6 +31,8 @@ import ( "strconv" "strings" + "github.com/flomesh-io/fsm/pkg/constants" + appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -164,9 +166,14 @@ func (r *serviceReconciler) newDaemonSet(ctx context.Context, svc *corev1.Servic Name: name, Namespace: svc.Namespace, Labels: map[string]string{ - nodeSelectorLabel: "false", - svcNameLabel: svc.Name, - svcNamespaceLabel: svc.Namespace, + nodeSelectorLabel: "false", + svcNameLabel: svc.Name, + svcNamespaceLabel: svc.Namespace, + constants.FSMAppNameLabelKey: constants.FSMAppNameLabelValue, + constants.FSMAppInstanceLabelKey: r.fctx.MeshName, + constants.FSMAppVersionLabelKey: r.fctx.FSMVersion, + constants.AppLabel: constants.FSMServiceLBName, + "meshName": r.fctx.MeshName, }, }, TypeMeta: metav1.TypeMeta{ diff --git a/pkg/gateway/client.go b/pkg/gateway/client.go index 4bc91b4fd..5c40a70ef 100644 --- a/pkg/gateway/client.go +++ b/pkg/gateway/client.go @@ -47,7 +47,7 @@ func newClient(informerCollection *informers.InformerCollection, kubeClient kube constants.FSMAppNameLabelKey: constants.FSMAppNameLabelValue, constants.FSMAppInstanceLabelKey: meshName, constants.FSMAppVersionLabelKey: fsmVersion, - constants.AppLabel: constants.FSMControllerName, + constants.AppLabel: constants.FSMGatewayName, }, }, Spec: gwv1beta1.GatewayClassSpec{ diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 2250b9f5f..fd1cfd0d7 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -30,13 +30,24 @@ import ( "context" "fmt" "io" + "path/filepath" + "regexp" + "sort" + "strings" "time" + "k8s.io/apimachinery/pkg/api/errors" + + "k8s.io/apimachinery/pkg/api/meta" + + "k8s.io/client-go/dynamic" + + "helm.sh/helm/v3/pkg/releaseutil" + "helm.sh/helm/v3/pkg/action" helm "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" - "helm.sh/helm/v3/pkg/release" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilyaml "k8s.io/apimachinery/pkg/util/yaml" @@ -50,16 +61,14 @@ import ( // RenderChart renders a chart and returns the rendered manifest func RenderChart( - releaseName string, + templateClient *helm.Install, object metav1.Object, chartSource []byte, mc configurator.Configurator, client client.Client, scheme *runtime.Scheme, - kubeVersion *chartutil.KubeVersion, resolveValues func(metav1.Object, configurator.Configurator) (map[string]interface{}, error), ) (ctrl.Result, error) { - installClient := helmClient(releaseName, object.GetNamespace(), kubeVersion) chart, err := loader.LoadArchive(bytes.NewReader(chartSource)) if err != nil { return ctrl.Result{}, fmt.Errorf("error loading chart for installation: %s", err) @@ -72,14 +81,16 @@ func RenderChart( } log.Debug().Msgf("[HELM UTIL] Values = %s", values) - rel, err := installClient.Run(chart, values) + rel, err := templateClient.Run(chart, values) if err != nil { - log.Error().Msgf("[HELM UTIL] Error installing chart: %s", err) - return ctrl.Result{}, fmt.Errorf("error install %s/%s: %s", object.GetNamespace(), object.GetName(), err) + log.Error().Msgf("[HELM UTIL] Error rendering chart: %s", err) + return ctrl.Result{}, fmt.Errorf("error rendering templates: %s", err) } - log.Debug().Msgf("[HELM UTIL] Manifest = \n%s\n", rel.Manifest) - if result, err := applyChartYAMLs(object, rel, client, scheme); err != nil { + manifests := rel.Manifest + log.Debug().Msgf("[HELM UTIL] Manifest = \n%s\n", manifests) + + if result, err := applyChartYAMLs(object, manifests, client, scheme); err != nil { log.Error().Msgf("[HELM UTIL] Error applying chart YAMLs: %s", err) return result, err } @@ -87,15 +98,9 @@ func RenderChart( return ctrl.Result{}, nil } -func helmClient(releaseName, namespace string, kubeVersion *chartutil.KubeVersion) *helm.Install { - configFlags := &genericclioptions.ConfigFlags{Namespace: &namespace} - - log.Debug().Msgf("[HELM UTIL] Initializing Helm Action Config ...") - actionConfig := new(action.Configuration) - _ = actionConfig.Init(configFlags, namespace, "secret", log.Debug().Msgf) - - log.Debug().Msgf("[HELM UTIL] Creating Helm Install Client ...") - installClient := helm.NewInstall(actionConfig) +func TemplateClient(cfg *helm.Configuration, releaseName, namespace string, kubeVersion *chartutil.KubeVersion) *helm.Install { + //log.Debug().Msgf("[HELM UTIL] Creating Helm Install Client ...") + installClient := helm.NewInstall(cfg) installClient.ReleaseName = releaseName installClient.Namespace = namespace installClient.CreateNamespace = false @@ -106,8 +111,17 @@ func helmClient(releaseName, namespace string, kubeVersion *chartutil.KubeVersio return installClient } -func applyChartYAMLs(owner metav1.Object, rel *release.Release, client client.Client, scheme *runtime.Scheme) (ctrl.Result, error) { - yamlReader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader([]byte(rel.Manifest)))) +func ActionConfig(namespace string, debugLog action.DebugLog) *helm.Configuration { + configFlags := &genericclioptions.ConfigFlags{Namespace: &namespace} + + actionConfig := new(action.Configuration) + _ = actionConfig.Init(configFlags, namespace, "secret", debugLog) + + return actionConfig +} + +func applyChartYAMLs(owner metav1.Object, manifests string, client client.Client, scheme *runtime.Scheme) (ctrl.Result, error) { + yamlReader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader([]byte(manifests)))) for { buf, err := yamlReader.Read() if err != nil { @@ -164,22 +178,95 @@ func isValidOwner(owner, object metav1.Object) bool { return true } -// MergeMaps merges two maps -func MergeMaps(a, b map[string]interface{}) map[string]interface{} { - out := make(map[string]interface{}, len(a)) - for k, v := range a { - out[k] = v +func ApplyYAMLs( + dynamicClient dynamic.Interface, + mapper meta.RESTMapper, + manifests string, + handler YAMLHandlerFunc, + showFiles ...string, +) error { + splitManifests := releaseutil.SplitManifests(manifests) + manifestsKeys := make([]string, 0, len(splitManifests)) + for k := range splitManifests { + manifestsKeys = append(manifestsKeys, k) } - for k, v := range b { - if v, ok := v.(map[string]interface{}); ok { - if bv, ok := out[k]; ok { - if bv, ok := bv.(map[string]interface{}); ok { - out[k] = MergeMaps(bv, v) + sort.Sort(releaseutil.BySplitManifestsOrder(manifestsKeys)) + + if len(showFiles) > 0 { + manifestNameRegex := regexp.MustCompile("# Source: [^/]+/(.+)") + var manifestsToRender []string + for _, f := range showFiles { + missing := true + // Use linux-style filepath separators to unify user's input path + f = filepath.ToSlash(f) + for _, manifestKey := range manifestsKeys { + manifest := splitManifests[manifestKey] + submatch := manifestNameRegex.FindStringSubmatch(manifest) + if len(submatch) == 0 { continue } + manifestName := submatch[1] + // manifest.Name is rendered using linux-style filepath separators on Windows as + // well as macOS/linux. + manifestPathSplit := strings.Split(manifestName, "/") + // manifest.Path is connected using linux-style filepath separators on Windows as + // well as macOS/linux + manifestPath := strings.Join(manifestPathSplit, "/") + + // if the filepath provided matches a manifest path in the + // chart, render that manifest + if matched, _ := filepath.Match(f, manifestPath); !matched { + continue + } + manifestsToRender = append(manifestsToRender, manifest) + missing = false + } + if missing { + return fmt.Errorf("could not find template %s in chart", f) } } - out[k] = v + for _, manifest := range manifestsToRender { + if err := handler(dynamicClient, mapper, manifest); err != nil { + return err + } + } + } else { + for _, manifestKey := range manifestsKeys { + manifest := splitManifests[manifestKey] + if err := handler(dynamicClient, mapper, manifest); err != nil { + return err + } + } + } + + return nil +} + +func ApplyManifest(dynamicClient dynamic.Interface, mapper meta.RESTMapper, manifest string) error { + obj, err := utils.DecodeYamlToUnstructured([]byte(manifest)) + if err != nil { + return err + } + + if err := utils.CreateOrUpdateUnstructured(context.TODO(), dynamicClient, mapper, obj); err != nil { + return err + } + + return nil +} + +func DeleteManifest(dynamicClient dynamic.Interface, mapper meta.RESTMapper, manifest string) error { + obj, err := utils.DecodeYamlToUnstructured([]byte(manifest)) + if err != nil { + return err } - return out + + if err := utils.DeleteUnstructured(context.TODO(), dynamicClient, mapper, obj); err != nil { + // ignore if not found + if !errors.IsNotFound(err) { + return err + } + } + + return nil } diff --git a/pkg/helm/types.go b/pkg/helm/types.go index f1148ee4f..7b3af6f7c 100644 --- a/pkg/helm/types.go +++ b/pkg/helm/types.go @@ -1,8 +1,15 @@ // Package helm provides utilities for helm package helm -import "github.com/flomesh-io/fsm/pkg/logger" +import ( + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/dynamic" + + "github.com/flomesh-io/fsm/pkg/logger" +) var ( log = logger.New("helm-utilities") ) + +type YAMLHandlerFunc func(dynamicClient dynamic.Interface, mapper meta.RESTMapper, manifest string) error diff --git a/pkg/sidecar/providers/pipy/repo/plugin.go b/pkg/sidecar/providers/pipy/repo/plugin.go index 7555b02d5..bffce2af9 100644 --- a/pkg/sidecar/providers/pipy/repo/plugin.go +++ b/pkg/sidecar/providers/pipy/repo/plugin.go @@ -169,7 +169,7 @@ func walkPluginConfig(cataloger catalog.MeshCataloger, plugin2MountPoint2Config continue } for mountPoint := range *mountPoint2ConfigItem { - (*mountPoint2ConfigItem)[mountPoint] = &pluginConfig.Config + (*mountPoint2ConfigItem)[mountPoint] = &pluginConfig.Config // #nosec G601 } for _, destinationRef := range pluginConfig.DestinationRefs { if destinationRef.Kind == policyv1alpha1.KindService { diff --git a/pkg/utils/ctrl.go b/pkg/utils/ctrl.go index 98a30817a..f2c6cc0b5 100644 --- a/pkg/utils/ctrl.go +++ b/pkg/utils/ctrl.go @@ -4,6 +4,14 @@ import ( "context" "reflect" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/api/meta" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -14,7 +22,7 @@ import ( func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object) (controllerutil.OperationResult, error) { // a copy of new object modifiedObj := obj.DeepCopyObject().(client.Object) - log.Info().Msgf("Modified: %v", modifiedObj) + log.Debug().Msgf("Modified: %v", modifiedObj) key := client.ObjectKeyFromObject(obj) gvk := obj.GetObjectKind().GroupVersionKind() @@ -23,26 +31,26 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object) (co log.Error().Msgf("Get Object %v, %s err: %s", gvk, key, err) return controllerutil.OperationResultNone, err } - log.Info().Msgf("Creating Object %v, %s ...", gvk, key) + log.Debug().Msgf("Creating Object %v, %s ...", gvk, key) if err := c.Create(ctx, obj); err != nil { log.Error().Msgf("Create Object %s err: %s", key, err) return controllerutil.OperationResultNone, err } - log.Info().Msgf("Object %v, %s is created successfully.", gvk, key) + log.Debug().Msgf("Object %v, %s is created successfully.", gvk, key) return controllerutil.OperationResultCreated, nil } - log.Info().Msgf("Found Object %v, %s: %v", gvk, key, obj) + log.Debug().Msgf("Found Object %v, %s: %v", gvk, key, obj) result := controllerutil.OperationResultNone if !reflect.DeepEqual(obj, modifiedObj) { - log.Info().Msgf("Patching Object %v, %s ...", gvk, key) + log.Debug().Msgf("Patching Object %v, %s ...", gvk, key) patchData, err := client.Merge.Data(modifiedObj) if err != nil { log.Error().Msgf("Create ApplyPatch err: %s", err) return controllerutil.OperationResultNone, err } - log.Info().Msgf("Patch data = \n\n%s\n\n", string(patchData)) + log.Debug().Msgf("Patch data = \n\n%s\n\n", string(patchData)) // Only issue a Patch if the before and after resources differ if err := c.Patch( @@ -57,6 +65,105 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object) (co result = controllerutil.OperationResultUpdated } - log.Info().Msgf("Object %v, %s is %s successfully.", gvk, key, result) + log.Debug().Msgf("Object %v, %s is %s successfully.", gvk, key, result) return result, nil } + +func CreateOrUpdateUnstructured(ctx context.Context, dynamicClient dynamic.Interface, mapper meta.RESTMapper, obj *unstructured.Unstructured) error { + // a copy of new object + modifiedObj := obj.DeepCopyObject().(*unstructured.Unstructured) + //log.Debug().Msgf("Modified: %v", modifiedObj) + + //key := client.ObjectKeyFromObject(obj) + //gvk := obj.GetObjectKind().GroupVersionKind() + + oldObj, err := getUnstructured(ctx, dynamicClient, mapper, obj) + if err != nil { + if !apierrors.IsNotFound(err) { + //log.Error().Msgf("Get Object %v, %s err: %s", gvk, key, err) + return err + } + + //log.Debug().Msgf("Creating Object %v, %s/%s ...", gvk, obj.GetNamespace(), obj.GetName()) + if _, err := createUnstructured(ctx, dynamicClient, mapper, obj); err != nil { + //log.Error().Msgf("Create Object %s err: %s", key, err) + return err + } + + //log.Debug().Msgf("Object %v, %s is created successfully.", gvk, key) + + return nil + } + //log.Debug().Msgf("Found Object %v, %s: %v", gvk, key, oldObj) + + if !reflect.DeepEqual(oldObj, modifiedObj) { + //log.Debug().Msgf("Patching Object %v, %s/%s ...", gvk, obj.GetNamespace(), obj.GetName()) + if _, err := patchUnstructured(ctx, dynamicClient, mapper, obj, modifiedObj); err != nil { + //log.Error().Msgf("Patch Object %v, %s err: %s", gvk, key, err) + return err + } + } + + return nil +} + +func getUnstructured(ctx context.Context, dynamicClient dynamic.Interface, mapper meta.RESTMapper, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + dri, err := getDynamicResourceInterface(obj, mapper, dynamicClient) + if err != nil { + return nil, err + } + + return dri.Get(ctx, obj.GetName(), metav1.GetOptions{}) +} + +func createUnstructured(ctx context.Context, dynamicClient dynamic.Interface, mapper meta.RESTMapper, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + dri, err := getDynamicResourceInterface(obj, mapper, dynamicClient) + if err != nil { + return nil, err + } + + return dri.Create(ctx, obj, metav1.CreateOptions{}) +} + +func patchUnstructured(ctx context.Context, dynamicClient dynamic.Interface, mapper meta.RESTMapper, obj, modifiedObj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + dri, err := getDynamicResourceInterface(obj, mapper, dynamicClient) + if err != nil { + return nil, err + } + + patchData, err := client.Merge.Data(modifiedObj) + if err != nil { + //log.Error().Msgf("Create ApplyPatch err: %s", err) + return nil, err + } + //log.Debug().Msgf("Patch data = %s", string(patchData)) + + // Only issue a Patch if the before and after resources differ + return dri.Patch(ctx, obj.GetName(), types.MergePatchType, patchData, metav1.PatchOptions{FieldManager: "fsm"}) +} + +func DeleteUnstructured(ctx context.Context, dynamicClient dynamic.Interface, mapper meta.RESTMapper, obj *unstructured.Unstructured) error { + dri, err := getDynamicResourceInterface(obj, mapper, dynamicClient) + if err != nil { + return err + } + + return dri.Delete(ctx, obj.GetName(), metav1.DeleteOptions{}) +} + +func getDynamicResourceInterface(obj *unstructured.Unstructured, mapper meta.RESTMapper, dynamicClient dynamic.Interface) (dynamic.ResourceInterface, error) { + gvk := obj.GetObjectKind().GroupVersionKind() + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, err + } + + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + if obj.GetNamespace() == "" { + obj.SetNamespace(corev1.NamespaceDefault) + } + return dynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace()), nil + } + + return dynamicClient.Resource(mapping.Resource), nil +} diff --git a/pkg/webhook/gateway/gateway_webhook.go b/pkg/webhook/gateway/gateway_webhook.go index c01e4f04d..d97205011 100644 --- a/pkg/webhook/gateway/gateway_webhook.go +++ b/pkg/webhook/gateway/gateway_webhook.go @@ -29,6 +29,8 @@ import ( "fmt" "net/http" + gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + admissionregv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -47,6 +49,7 @@ import ( type register struct { *webhook.RegisterConfig + gatewayAPIClient gatewayApiClientset.Interface } const ( @@ -56,7 +59,8 @@ const ( // NewRegister creates a new gateway webhook register func NewRegister(cfg *webhook.RegisterConfig) webhook.Register { return ®ister{ - RegisterConfig: cfg, + RegisterConfig: cfg, + gatewayAPIClient: gatewayApiClientset.NewForConfigOrDie(cfg.KubeConfig), } } @@ -95,20 +99,25 @@ func (r *register) GetWebhooks() ([]admissionregv1.MutatingWebhook, []admissionr // GetHandlers returns the handlers to be registered of gateway func (r *register) GetHandlers() map[string]http.Handler { return map[string]http.Handler{ - constants.GatewayMutatingWebhookPath: webhook.DefaultingWebhookFor(newDefaulter(r.KubeClient, r.Config)), + constants.GatewayMutatingWebhookPath: webhook.DefaultingWebhookFor(newDefaulter(r.KubeClient, r.gatewayAPIClient, r.Config, r.MeshName, r.FSMVersion)), constants.GatewayValidatingWebhookPath: webhook.ValidatingWebhookFor(newValidator(r.KubeClient)), } } type defaulter struct { - kubeClient kubernetes.Interface - cfg configurator.Configurator + kubeClient kubernetes.Interface + gatewayAPIClient gatewayApiClientset.Interface + cfg configurator.Configurator + meshName string + fsmVersion string } -func newDefaulter(kubeClient kubernetes.Interface, cfg configurator.Configurator) *defaulter { +func newDefaulter(kubeClient kubernetes.Interface, gatewayAPIClient gatewayApiClientset.Interface, cfg configurator.Configurator, meshName, fsmVersion string) *defaulter { return &defaulter{ kubeClient: kubeClient, cfg: cfg, + meshName: meshName, + fsmVersion: fsmVersion, } } @@ -127,11 +136,28 @@ func (w *defaulter) SetDefaults(obj interface{}) { log.Debug().Msgf("Default Webhook, name=%s", gateway.Name) log.Debug().Msgf("Before setting default values, spec=%v", gateway.Spec) - //meshConfig := w.configStore.MeshConfig.GetConfig() - // - //if meshConfig == nil { - // return - //} + gatewayClass, err := w.gatewayAPIClient. + GatewayV1beta1(). + GatewayClasses(). + Get(context.TODO(), string(gateway.Spec.GatewayClassName), metav1.GetOptions{}) + if err != nil { + log.Error().Msgf("failed to get gatewayclass %s", gateway.Spec.GatewayClassName) + return + } + + if gatewayClass.Spec.ControllerName != constants.GatewayController { + log.Warn().Msgf("class controller of Gateway %s/%s is not %s", gateway.Namespace, gateway.Name, constants.GatewayController) + return + } + + // if it's a valid gateway, set default values + if len(gateway.Labels) == 0 { + gateway.Labels = map[string]string{} + } + gateway.Labels[constants.FSMAppNameLabelKey] = constants.FSMAppNameLabelValue + gateway.Labels[constants.FSMAppInstanceLabelKey] = w.meshName + gateway.Labels[constants.FSMAppVersionLabelKey] = w.fsmVersion + gateway.Labels[constants.AppLabel] = constants.FSMGatewayName log.Debug().Msgf("After setting default values, spec=%v", gateway.Spec) } diff --git a/pkg/webhook/namespacedingress/namespacedingress_webhook.go b/pkg/webhook/namespacedingress/namespacedingress_webhook.go index 6cbebd050..9a6bb9599 100644 --- a/pkg/webhook/namespacedingress/namespacedingress_webhook.go +++ b/pkg/webhook/namespacedingress/namespacedingress_webhook.go @@ -92,7 +92,7 @@ func (r *register) GetWebhooks() ([]admissionregv1.MutatingWebhook, []admissionr // GetHandlers returns the handlers for the namespacedingress resources func (r *register) GetHandlers() map[string]http.Handler { return map[string]http.Handler{ - constants.NamespacedIngressMutatingWebhookPath: webhook.DefaultingWebhookFor(newDefaulter(r.KubeClient, r.Config)), + constants.NamespacedIngressMutatingWebhookPath: webhook.DefaultingWebhookFor(newDefaulter(r.KubeClient, r.Config, r.MeshName, r.FSMVersion)), constants.NamespacedIngressValidatingWebhookPath: webhook.ValidatingWebhookFor(newValidator(r.KubeClient, r.nsigClient)), } } @@ -100,12 +100,16 @@ func (r *register) GetHandlers() map[string]http.Handler { type defaulter struct { kubeClient kubernetes.Interface cfg configurator.Configurator + meshName string + fsmVersion string } -func newDefaulter(kubeClient kubernetes.Interface, cfg configurator.Configurator) *defaulter { +func newDefaulter(kubeClient kubernetes.Interface, cfg configurator.Configurator, meshName, fsmVersion string) *defaulter { return &defaulter{ kubeClient: kubeClient, cfg: cfg, + meshName: meshName, + fsmVersion: fsmVersion, } } @@ -129,6 +133,13 @@ func (w *defaulter) SetDefaults(obj interface{}) { //if meshConfig == nil { // return //} + if len(c.Labels) == 0 { + c.Labels = map[string]string{} + } + c.Labels[constants.FSMAppNameLabelKey] = constants.FSMAppNameLabelValue + c.Labels[constants.FSMAppInstanceLabelKey] = w.meshName + c.Labels[constants.FSMAppVersionLabelKey] = w.fsmVersion + c.Labels[constants.AppLabel] = constants.FSMIngressName if c.Spec.ServiceAccountName == "" { c.Spec.ServiceAccountName = "fsm-namespaced-ingress"