From 103346111b5b6e88a47e5185ab254457377189ae Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Fri, 22 Nov 2024 18:21:06 +0800 Subject: [PATCH] add agentcsrapproving controller to auto approve agent csr Signed-off-by: zhzhuang-zju --- .../app/controllermanager.go | 13 +- .../approver/agent_csr_approving.go | 293 ++++++++++++++++++ pkg/util/certificate/csr.go | 53 ++++ pkg/util/certificate/csr_test.go | 78 +++++ 4 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 pkg/controllers/certificate/approver/agent_csr_approving.go create mode 100644 pkg/util/certificate/csr.go create mode 100644 pkg/util/certificate/csr_test.go diff --git a/cmd/controller-manager/app/controllermanager.go b/cmd/controller-manager/app/controllermanager.go index 1bf481b09965..e49559ea57e5 100644 --- a/cmd/controller-manager/app/controllermanager.go +++ b/cmd/controller-manager/app/controllermanager.go @@ -51,6 +51,7 @@ import ( "github.com/karmada-io/karmada/pkg/clusterdiscovery/clusterapi" "github.com/karmada-io/karmada/pkg/controllers/applicationfailover" "github.com/karmada-io/karmada/pkg/controllers/binding" + "github.com/karmada-io/karmada/pkg/controllers/certificate/approver" "github.com/karmada-io/karmada/pkg/controllers/cluster" controllerscontext "github.com/karmada-io/karmada/pkg/controllers/context" "github.com/karmada-io/karmada/pkg/controllers/cronfederatedhpa" @@ -209,7 +210,7 @@ func Run(ctx context.Context, opts *options.Options) error { var controllers = make(controllerscontext.Initializers) // controllersDisabledByDefault is the set of controllers which is disabled by default -var controllersDisabledByDefault = sets.New("hpaScaleTargetMarker", "deploymentReplicasSyncer") +var controllersDisabledByDefault = sets.New("hpaScaleTargetMarker", "deploymentReplicasSyncer", "agentcsrapproving") func init() { controllers["cluster"] = startClusterController @@ -236,6 +237,7 @@ func init() { controllers["endpointsliceDispatch"] = startEndpointSliceDispatchController controllers["remedy"] = startRemedyController controllers["workloadRebalancer"] = startWorkloadRebalancerController + controllers["agentcsrapproving"] = startAgentCSRApprovingController } func startClusterController(ctx controllerscontext.Context) (enabled bool, err error) { @@ -723,6 +725,15 @@ func startWorkloadRebalancerController(ctx controllerscontext.Context) (enabled return true, nil } +func startAgentCSRApprovingController(ctx controllerscontext.Context) (enabled bool, err error) { + agentCSRApprover := approver.AgentCSRApprovingController{Client: ctx.KubeClientSet} + err = agentCSRApprover.SetupWithManager(ctx.Mgr) + if err != nil { + return false, err + } + return true, nil +} + // setupControllers initialize controllers and setup one by one. func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stopChan <-chan struct{}) { restConfig := mgr.GetConfig() diff --git a/pkg/controllers/certificate/approver/agent_csr_approving.go b/pkg/controllers/certificate/approver/agent_csr_approving.go new file mode 100644 index 000000000000..f42c2db62d60 --- /dev/null +++ b/pkg/controllers/certificate/approver/agent_csr_approving.go @@ -0,0 +1,293 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package approver + +import ( + "context" + "crypto/x509" + "fmt" + "reflect" + "strings" + + authorizationv1 "k8s.io/api/authorization/v1" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/karmada-io/karmada/pkg/util/certificate" +) + +const ( + csrApprovingController = "agent-csr-approving-controller" + agentCSRGroup = "system:karmada:agents" + agentCSRUserPrefix = "system:karmada:agent:" +) + +// AgentCSRApprovingController is used to automatically approve the agent's CSR. +type AgentCSRApprovingController struct { + Client kubernetes.Interface +} + +// Reconcile performs a full reconciliation for the object referred to by the Request. +// The Controller will requeue the Request to be processed again if an error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (a *AgentCSRApprovingController) Reconcile(ctx context.Context, req controllerruntime.Request) (controllerruntime.Result, error) { + klog.V(4).Infof("Reconciling for CertificateSigningRequest %s", req.Name) + + // 1. get latest CertificateSigningRequest + var csr *certificatesv1.CertificateSigningRequest + var err error + if csr, err = a.Client.CertificatesV1().CertificateSigningRequests().Get(ctx, req.Name, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + klog.Infof("no need to reconcile CertificateSigningRequest %s for it not found", req.Name) + return controllerruntime.Result{}, nil + } + return controllerruntime.Result{}, err + } + + if csr.DeletionTimestamp != nil { + klog.Infof("no need to reconcile CertificateSigningRequest %s for it has been deleted", csr.Name) + return controllerruntime.Result{}, nil + } + + // 2. auto approve csr if it is an agent csr and passes authentication. + err = a.handleCertificateSigningRequest(ctx, csr) + if err != nil { + return controllerruntime.Result{}, err + } + + return controllerruntime.Result{}, nil +} + +func (a *AgentCSRApprovingController) handleCertificateSigningRequest(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) error { + if len(csr.Status.Certificate) != 0 { + return nil + } + if approved, denied := certificate.GetCertApprovalCondition(&csr.Status); approved || denied { + return nil + } + x509cr, err := certificate.ParseCSR(csr.Spec.Request) + if err != nil { + return fmt.Errorf("unable to parse csr %q: %v", csr.Name, err) + } + var tried []string + + for _, r := range agentCSRRecognizers() { + if !r.recognize(csr, x509cr) { + continue + } + + tried = append(tried, r.permission.Subresource) + + approved, err := a.authorize(ctx, csr, r.permission) + if err != nil { + return err + } + if approved { + appendApprovalCondition(csr, r.successMessage) + _, err = a.Client.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error updating approval for csr %s: %v", csr.Name, err) + } + return nil + } + } + + if len(tried) != 0 { + klog.Warningf("recognized csr %q as %v but subject access review was not approved", csr.Name, tried) + } + + return nil +} + +func (a *AgentCSRApprovingController) authorize(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, rattrs authorizationv1.ResourceAttributes) (bool, error) { + extra := make(map[string]authorizationv1.ExtraValue) + for k, v := range csr.Spec.Extra { + extra[k] = authorizationv1.ExtraValue(v) + } + + sar := &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + User: csr.Spec.Username, + UID: csr.Spec.UID, + Groups: csr.Spec.Groups, + Extra: extra, + ResourceAttributes: &rattrs, + }, + } + sar, err := a.Client.AuthorizationV1().SubjectAccessReviews().Create(ctx, sar, metav1.CreateOptions{}) + if err != nil { + return false, err + } + return sar.Status.Allowed, nil +} + +func isIssuedByKubeAPIServerClientSigner(csr *certificatesv1.CertificateSigningRequest) bool { + return csr.Spec.SignerName == certificatesv1.KubeAPIServerClientSignerName +} + +// csrRecognizer used to identify whether the CSRs is the target CSRs and to perform authentication. +type csrRecognizer struct { + // recognize identifies whether the CSRs is the target CSRs + recognize func(csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) bool + // permission used to indicate the permissions required for auto csr approving. + permission authorizationv1.ResourceAttributes + // successMessage contains a human-readable message with details if auto csr approving is successful. + successMessage string +} + +// agentCSRRecognizers used to identify whether the CSRs is the agent CSRs and to perform authentication. +func agentCSRRecognizers() []csrRecognizer { + recognizers := []csrRecognizer{ + { + recognize: isSelfAgentCSR, + permission: authorizationv1.ResourceAttributes{Group: "certificates.k8s.io", Resource: "certificatesigningrequests", Verb: "create", Subresource: "selfclusteragent", Version: "*"}, + successMessage: "Auto approving self karmada agent certificate after SubjectAccessReview.", + }, + { + recognize: isAgentCSR, + permission: authorizationv1.ResourceAttributes{Group: "certificates.k8s.io", Resource: "certificatesigningrequests", Verb: "create", Subresource: "clusteragent", Version: "*"}, + successMessage: "Auto approving karmada agent certificate after SubjectAccessReview.", + }, + } + return recognizers +} + +func appendApprovalCondition(csr *certificatesv1.CertificateSigningRequest, message string) { + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + Reason: "AutoApproved", + Message: message, + }) +} + +// isAgentCSR determines if the provided csr is an agent csr. +// Agent csr is created for karmada-agent by bootstrap token during the cluster registering process. +// The 'signer' field must be set to "kubernetes.io/kube-apiserver-client". +// The 'Organization' field in the CertificateRequest must be "system:agents". +// The 'CommonName' must be prefixed with "system:agent:". +func isAgentCSR(csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) bool { + if csr.Spec.SignerName != certificatesv1.KubeAPIServerClientSignerName { + return false + } + + return ValidateAgentCSR(x509cr, usagesToSet(csr.Spec.Usages)) == nil +} + +// isSelfAgentCSR determines if the provided csr is a self-agent csr. +// Self-agent csr is created by karmada-agent to enable certificate rotation feature. +// In contrast to the agent CSR, for a self-agent CSR, the username of the user who creates the `CertificateSigningRequest` must be identical to the 'CommonName' specified in the CertificateRequest. +func isSelfAgentCSR(csr *certificatesv1.CertificateSigningRequest, x509cr *x509.CertificateRequest) bool { + if csr.Spec.Username != x509cr.Subject.CommonName { + return false + } + return isAgentCSR(csr, x509cr) +} + +var ( + errOrganizationNotSystemAgents = fmt.Errorf("subject organization is not system:karmada:agents") + errCommonNameNotSystemAgent = fmt.Errorf("subject common name does not begin with system:karmada:agent: prefix") + errDNSSANNotAllowed = fmt.Errorf("DNS subjectAltNames are not allowed") + errEmailSANNotAllowed = fmt.Errorf("email subjectAltNames are not allowed") + errIPSANNotAllowed = fmt.Errorf("IP subjectAltNames are not allowed") + errURISANNotAllowed = fmt.Errorf("URI subjectAltNames are not allowed") +) + +// ValidateAgentCSR used to determine if the CSR is a valid agent's CSR. +func ValidateAgentCSR(req *x509.CertificateRequest, usages sets.Set[string]) error { + if !reflect.DeepEqual([]string{agentCSRGroup}, req.Subject.Organization) { + return errOrganizationNotSystemAgents + } + + if len(req.DNSNames) > 0 { + return errDNSSANNotAllowed + } + + if len(req.EmailAddresses) > 0 { + return errEmailSANNotAllowed + } + + if len(req.IPAddresses) > 0 { + return errIPSANNotAllowed + } + + if len(req.URIs) > 0 { + return errURISANNotAllowed + } + + if !strings.HasPrefix(req.Subject.CommonName, agentCSRUserPrefix) { + return errCommonNameNotSystemAgent + } + + if !agentRequiredUsages.Equal(usages) && !agentRequiredUsagesNoKeyEncipherment.Equal(usages) { + return fmt.Errorf("usages did not match %v", sets.List(agentRequiredUsages)) + } + + return nil +} + +var ( + agentRequiredUsagesNoKeyEncipherment = sets.New[string]( + string(certificatesv1.UsageDigitalSignature), + string(certificatesv1.UsageClientAuth), + ) + agentRequiredUsages = sets.New[string]( + string(certificatesv1.UsageDigitalSignature), + string(certificatesv1.UsageKeyEncipherment), + string(certificatesv1.UsageClientAuth), + ) +) + +func usagesToSet(usages []certificatesv1.KeyUsage) sets.Set[string] { + result := sets.New[string]() + for _, usage := range usages { + result.Insert(string(usage)) + } + return result +} + +// SetupWithManager creates a controller and registers to controller manager. +func (a *AgentCSRApprovingController) SetupWithManager(mgr controllerruntime.Manager) error { + var predicateFunc = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + csr := e.Object.(*certificatesv1.CertificateSigningRequest) + // agent certificate is signed by "kubernetes.io/kube-apiserver-Client" signer + return isIssuedByKubeAPIServerClientSigner(csr) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + newCSR := e.ObjectNew.(*certificatesv1.CertificateSigningRequest) + // agent certificate is signed by "kubernetes.io/kube-apiserver-Client" signer + return isIssuedByKubeAPIServerClientSigner(newCSR) + }, + DeleteFunc: func(event.DeleteEvent) bool { return false }, + GenericFunc: func(event.GenericEvent) bool { return false }, + } + + return controllerruntime.NewControllerManagedBy(mgr). + Named(csrApprovingController). + For(&certificatesv1.CertificateSigningRequest{}, builder.WithPredicates(predicateFunc)). + Complete(a) +} diff --git a/pkg/util/certificate/csr.go b/pkg/util/certificate/csr.go new file mode 100644 index 000000000000..92c523c7b40b --- /dev/null +++ b/pkg/util/certificate/csr.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + + certificatesv1 "k8s.io/api/certificates/v1" +) + +const certificateRequest = "CERTIFICATE REQUEST" + +// ParseCSR extracts the CSR from the bytes and decodes it. +func ParseCSR(pemBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != certificateRequest { + return nil, fmt.Errorf("PEM block type must be CERTIFICATE REQUEST") + } + csr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil { + return nil, err + } + return csr, nil +} + +// GetCertApprovalCondition return true if the status conditions of csr is Approved or Denied. +func GetCertApprovalCondition(status *certificatesv1.CertificateSigningRequestStatus) (approved bool, denied bool) { + for _, c := range status.Conditions { + if c.Type == certificatesv1.CertificateApproved { + approved = true + } + if c.Type == certificatesv1.CertificateDenied { + denied = true + } + } + return +} diff --git a/pkg/util/certificate/csr_test.go b/pkg/util/certificate/csr_test.go new file mode 100644 index 000000000000..d041a7f3c19e --- /dev/null +++ b/pkg/util/certificate/csr_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + certificatesv1 "k8s.io/api/certificates/v1" +) + +func TestGetCertApprovalCondition(t *testing.T) { + testItems := []struct { + name string + status *certificatesv1.CertificateSigningRequestStatus + approved bool + denied bool + }{ + { + name: "csr has been approved", + status: &certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + {Type: certificatesv1.CertificateApproved}, + }, + }, + approved: true, + denied: false, + }, + { + name: "csr has been denied", + status: &certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + {Type: certificatesv1.CertificateDenied}, + }, + }, + approved: false, + denied: true, + }, + { + name: "the signer failed to issue the certificate", + status: &certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + {Type: certificatesv1.CertificateFailed}, + }, + }, + approved: false, + denied: false, + }, + { + name: "csr with no conditions", + status: &certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{}, + }, + approved: false, + denied: false, + }, + } + + for _, item := range testItems { + approved, denied := GetCertApprovalCondition(item.status) + assert.Equal(t, item.approved, approved) + assert.Equal(t, item.denied, denied) + } +}