Add SingleStore TLS (#97)

Signed-off-by: ashraful <[email protected]>

go.mod

@@ -26,7 +26,7 @@ require (
 	k8s.io/klog/v2 v2.120.1
 	kmodules.xyz/client-go v0.29.13
 	kmodules.xyz/custom-resources v0.29.1
-	kubedb.dev/apimachinery v0.44.1-0.20240425042236-6efef42b8792
+	kubedb.dev/apimachinery v0.44.1-0.20240426055822-7fb3d5619cd2
 	sigs.k8s.io/controller-runtime v0.17.2
 	xorm.io/xorm v1.3.6
 )

go.sum

@@ -609,8 +609,8 @@ kmodules.xyz/monitoring-agent-api v0.29.0 h1:gpFl6OZrlMLb/ySMHdREI9EwGtnJ91oZBn9
 kmodules.xyz/monitoring-agent-api v0.29.0/go.mod h1:iNbvaMTgVFOI5q2LJtGK91j4Dmjv4ZRiRdasGmWLKQI=
 kmodules.xyz/offshoot-api v0.29.1 h1:Pm83nzYHbqfCYKPCHrK0io387yXTaBmSydoAP6nF0WU=
 kmodules.xyz/offshoot-api v0.29.1/go.mod h1:SeGhKGXxNAy56cLnskEcLgCH+LRFN+MhJzvrZzPqUlM=
-kubedb.dev/apimachinery v0.44.1-0.20240425042236-6efef42b8792 h1:WNzbq7rB18pla0OkJszSg1eWZ2/VNZmdf6YNq97WRSU=
-kubedb.dev/apimachinery v0.44.1-0.20240425042236-6efef42b8792/go.mod h1:0uGwbmD4XN00LeU236LLOgoocK+UBoB9ojdstnZeJd8=
+kubedb.dev/apimachinery v0.44.1-0.20240426055822-7fb3d5619cd2 h1:Mv6PlqBRD3YimORjoC8f2VqGFNfGFgFHrmlpsNfZcug=
+kubedb.dev/apimachinery v0.44.1-0.20240426055822-7fb3d5619cd2/go.mod h1:0uGwbmD4XN00LeU236LLOgoocK+UBoB9ojdstnZeJd8=
 kubeops.dev/petset v0.0.5 h1:VVXi39JhjondlbHyZ98z0MLp6VCmiCMinL59K48Y2zA=
 kubeops.dev/petset v0.0.5/go.mod h1:ijtKT1HlAht2vBEZj5LW7C00XEs3B0d1VdCQgd5V4cA=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=

singlestore/kubedb_client_builder.go

@@ -18,10 +18,12 @@ package singlestore
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"database/sql"
 	"fmt"
 
-	_ "github.com/go-sql-driver/mysql"
+	sql_driver "github.com/go-sql-driver/mysql"
 	core "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 	api "kubedb.dev/apimachinery/apis/kubedb/v1alpha2"
@@ -147,10 +149,10 @@ func (o *KubeDBClientBuilder) getConnectionString() (string, error) {
 	}
 
 	tlsConfig := ""
-	/*if o.db.Spec.RequireSSL && o.db.Spec.TLS != nil {
+	if o.db.Spec.TLS != nil {
 		// get client-secret
 		var clientSecret core.Secret
-		err := o.kc.Get(o.ctx, client.ObjectKey{Namespace: o.db.GetNamespace(), Name: o.db.GetCertSecretName(api.MySQLClientCert)}, &clientSecret)
+		err := o.kc.Get(o.ctx, client.ObjectKey{Namespace: o.db.GetNamespace(), Name: o.db.GetCertSecretName(api.SinglestoreClientCert)}, &clientSecret)
 		if err != nil {
 			return "", err
 		}
@@ -168,19 +170,15 @@ func (o *KubeDBClientBuilder) getConnectionString() (string, error) {
 		clientCert = append(clientCert, cert)
 
 		// tls custom setup
-		if o.db.Spec.RequireSSL {
-			err = sql_driver.RegisterTLSConfig(api.MySQLTLSConfigCustom, &tls.Config{
-				RootCAs:      certPool,
-				Certificates: clientCert,
-			})
-			if err != nil {
-				return "", err
-			}
-			tlsConfig = fmt.Sprintf("tls=%s", api.MySQLTLSConfigCustom)
-		} else {
-			tlsConfig = fmt.Sprintf("tls=%s", api.MySQLTLSConfigSkipVerify)
+		err = sql_driver.RegisterTLSConfig(api.SinglestoreTLSConfigCustom, &tls.Config{
+			RootCAs:      certPool,
+			Certificates: clientCert,
+		})
+		if err != nil {
+			return "", err
 		}
-	}*/
+		tlsConfig = fmt.Sprintf("tls=%s", api.SinglestoreTLSConfigCustom)
+	}
 
 	connector := fmt.Sprintf("%v:%v@tcp(%s:%d)/%s?%s", user, pass, o.url, 3306, "memsql", tlsConfig)
 	return connector, nil

vendor/kubedb.dev/apimachinery/apis/kubedb/v1alpha2/constants.go

@@ -326,19 +326,22 @@ const (
 	SinglestoreDatabasePortName       = "db"
 	SinglestorePrimaryServicePortName = "primary"
 	SinglestoreStudioPortName         = "studio"
-	SinglestoreDatabasePort           = 3306
-	SinglestoreStudioPort             = 8081
-	SinglestoreExporterPort           = 9104
-	SinglestoreRootUserName           = "ROOT_USERNAME"
-	SinglestoreRootPassword           = "ROOT_PASSWORD"
-	SinglestoreRootUser               = "root"
-	DatabasePodMaster                 = "Master"
-	DatabasePodAggregator             = "Aggregator"
-	DatabasePodLeaf                   = "Leaf"
-	PetSetTypeAggregator              = "aggregator"
-	PetSetTypeLeaf                    = "leaf"
-	SinglestoreDatabaseHealth         = "singlestore_health"
-	SinglestoreTableHealth            = "singlestore_health_table"
+
+	SinglestoreDatabasePort = 3306
+	SinglestoreStudioPort   = 8081
+	SinglestoreExporterPort = 9104
+
+	SinglestoreRootUserName = "ROOT_USERNAME"
+	SinglestoreRootPassword = "ROOT_PASSWORD"
+	SinglestoreRootUser     = "root"
+	DatabasePodMaster       = "Master"
+	DatabasePodAggregator   = "Aggregator"
+	DatabasePodLeaf         = "Leaf"
+	PetSetTypeAggregator    = "aggregator"
+	PetSetTypeLeaf          = "leaf"
+
+	SinglestoreDatabaseHealth = "singlestore_health"
+	SinglestoreTableHealth    = "singlestore_health_table"
 
 	SinglestoreCoordinatorContainerName = "singlestore-coordinator"
 	SinglestoreContainerName            = "singlestore"
@@ -352,6 +355,14 @@ const (
 	SinglestoreVolumeMountPathInitScript     = "/scripts"
 	SinglestoreVolumeNameData                = "data"
 	SinglestoreVolumeMountPathData           = "/var/lib/memsql"
+	SinglestoreVolumeNameTLS                 = "tls-volume"
+	SinglestoreVolumeMountPathTLS            = "/etc/memsql/certs"
+
+	SinglestoreTLSConfigCustom     = "custom"
+	SinglestoreTLSConfigSkipVerify = "skip-verify"
+	SinglestoreTLSConfigTrue       = "true"
+	SinglestoreTLSConfigFalse      = "false"
+	SinglestoreTLSConfigPreferred  = "preferred"
 
 	// =========================== MSSQL Constants ============================
 	MSSQLSAUser                  = "sa"
@@ -560,10 +571,16 @@ const (
 	EnvPgpoolService                   = "PGPOOL_SERVICE"
 	EnvPgpoolServicePort               = "PGPOOL_SERVICE_PORT"
 	EnvPgpoolSSLMode                   = "SSLMODE"
+	EnvPgpoolExporterConnectionString  = "DATA_SOURCE_NAME"
 	PgpoolDefaultSSLMode               = "disable"
 	PgpoolExporterContainerName        = "exporter"
 	PgpoolAuthUsername                 = "pcp"
 	SyncPeriod                         = 10
+	PgpoolTlsVolumeName                = "certs"
+	PgpoolTlsVolumeMountPath           = "/config/tls"
+	PgpoolExporterTlsVolumeName        = "exporter-certs"
+	PgpoolExporterTlsVolumeMountPath   = "/tls/certs"
+	PgpoolRootUser                     = "postgres"
 	// ========================================== ZooKeeper Constants =================================================//
 
 	KubeDBZooKeeperRoleName         = "kubedb:zookeeper-version-reader"

vendor/kubedb.dev/apimachinery/apis/kubedb/v1alpha2/pgpool_helpers.go

@@ -19,6 +19,7 @@ package v1alpha2
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"kubedb.dev/apimachinery/apis"
 	catalog "kubedb.dev/apimachinery/apis/catalog/v1alpha1"
@@ -32,10 +33,12 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
+	kmapi "kmodules.xyz/client-go/api/v1"
 	"kmodules.xyz/client-go/apiextensions"
 	core_util "kmodules.xyz/client-go/core/v1"
 	meta_util "kmodules.xyz/client-go/meta"
 	"kmodules.xyz/client-go/policy/secomp"
+	appcat "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	mona "kmodules.xyz/monitoring-agent-api/api/v1"
 	ofst "kmodules.xyz/offshoot-api/api/v2"
 	pslister "kubeops.dev/petset/client/listers/apps/v1"
@@ -69,6 +72,10 @@ func (p *Pgpool) ConfigSecretName() string {
 	return meta_util.NameWithSuffix(p.OffshootName(), "config")
 }
 
+func (p *Pgpool) TLSSecretName() string {
+	return meta_util.NameWithSuffix(p.OffshootName(), "tls-certs")
+}
+
 func (p *Pgpool) ServiceAccountName() string {
 	return p.OffshootName()
 }
@@ -203,6 +210,66 @@ func (p *Pgpool) ServiceLabels(alias ServiceAlias, extraLabels ...map[string]str
 	return p.offshootLabels(meta_util.OverwriteKeys(p.OffshootSelectors(), extraLabels...), svcTemplate.Labels)
 }
 
+func (p *Pgpool) GetSSLMODE(appBinding *appcat.AppBinding) (PgpoolSSLMode, error) {
+	if appBinding.Spec.ClientConfig.Service == nil {
+		return PgpoolSSLModeDisable, nil
+	}
+	sslmodeString := appBinding.Spec.ClientConfig.Service.Query
+	if sslmodeString == "" {
+		return PgpoolSSLModeDisable, nil
+	}
+	temps := strings.Split(sslmodeString, "=")
+	if len(temps) != 2 {
+		return "", fmt.Errorf("the sslmode is not valid. please provide the valid template. the temlpate should be like this: sslmode=<your_desire_sslmode>")
+	}
+	return PgpoolSSLMode(strings.TrimSpace(temps[1])), nil
+}
+
+func (p *Pgpool) IsBackendTLSEnabled() (bool, error) {
+	apb := appcat.AppBinding{}
+	err := DefaultClient.Get(context.TODO(), types.NamespacedName{
+		Name:      p.Spec.PostgresRef.Name,
+		Namespace: p.Spec.PostgresRef.Namespace,
+	}, &apb)
+	if err != nil {
+		return false, err
+	}
+	sslMode, err := p.GetSSLMODE(&apb)
+	if err != nil {
+		return false, err
+	}
+	if apb.Spec.TLSSecret != nil || len(apb.Spec.ClientConfig.CABundle) > 0 || sslMode != PgpoolSSLModeDisable {
+		return true, nil
+	}
+	return false, nil
+}
+
+// CertificateName returns the default certificate name and/or certificate secret name for a certificate alias
+func (p *Pgpool) CertificateName(alias PgpoolCertificateAlias) string {
+	return meta_util.NameWithSuffix(p.Name, fmt.Sprintf("%s-cert", string(alias)))
+}
+
+// GetCertSecretName returns the secret name for a certificate alias if any provide,
+// otherwise returns default certificate secret name for the given alias.
+func (p *Pgpool) GetCertSecretName(alias PgpoolCertificateAlias) string {
+	if p.Spec.TLS != nil {
+		name, ok := kmapi.GetCertificateSecretName(p.Spec.TLS.Certificates, string(alias))
+		if ok {
+			return name
+		}
+	}
+	return p.CertificateName(alias)
+}
+
+func (p *Pgpool) SetTLSDefaults() {
+	if p.Spec.TLS == nil || p.Spec.TLS.IssuerRef == nil {
+		return
+	}
+	p.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(p.Spec.TLS.Certificates, string(PgpoolServerCert), p.CertificateName(PgpoolServerCert))
+	p.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(p.Spec.TLS.Certificates, string(PgpoolClientCert), p.CertificateName(PgpoolClientCert))
+	p.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(p.Spec.TLS.Certificates, string(PgpoolMetricsExporterCert), p.CertificateName(PgpoolMetricsExporterCert))
+}
+
 func (p *Pgpool) SetSecurityContext(ppVersion *catalog.PgpoolVersion, podTemplate *ofst.PodTemplateSpec) {
 	if podTemplate == nil {
 		return
@@ -272,6 +339,16 @@ func (p *Pgpool) SetDefaults() {
 		p.Spec.PodTemplate.Spec.Containers = []core.Container{}
 	}
 
+	if p.Spec.TLS != nil {
+		if p.Spec.SSLMode == "" {
+			p.Spec.SSLMode = PgpoolSSLModeVerifyFull
+		}
+	} else {
+		if p.Spec.SSLMode == "" {
+			p.Spec.SSLMode = PgpoolSSLModeDisable
+		}
+	}
+
 	ppVersion := catalog.PgpoolVersion{}
 	err := DefaultClient.Get(context.TODO(), types.NamespacedName{
 		Name: p.Spec.Version,
@@ -297,6 +374,7 @@ func (p *Pgpool) SetDefaults() {
 		}
 	}
 
+	p.SetTLSDefaults()
 	p.SetHealthCheckerDefaults()
 	p.SetSecurityContext(&ppVersion, p.Spec.PodTemplate)
 	p.setContainerResourceLimits(p.Spec.PodTemplate)

vendor/kubedb.dev/apimachinery/apis/kubedb/v1alpha2/pgpool_types.go

@@ -111,6 +111,17 @@ type PgpoolSpec struct {
 	// +kubebuilder:default={name: "default"}
 	// +optional
 	PodPlacementPolicy *core.LocalObjectReference `json:"podPlacementPolicy,omitempty"`
+
+	// SSLMode for both standalone and clusters. [disable;allow;prefer;require;verify-ca;verify-full]
+	SSLMode PgpoolSSLMode `json:"sslMode,omitempty"`
+
+	// ClientAuthMode for sidecar or sharding. (default will be md5. [md5;scram;cert])
+	// +kubebuilder:default=md5
+	ClientAuthMode PgpoolClientAuthMode `json:"clientAuthMode,omitempty"`
+
+	// TLS contains tls configurations for client and server.
+	// +optional
+	TLS *kmapi.TLSConfig `json:"tls,omitempty"`
 }
 
 // PgpoolStatus defines the observed state of Pgpool
@@ -148,3 +159,65 @@ type PgpoolList struct {
 	meta.ListMeta `json:"metadata,omitempty"`
 	Items         []Pgpool `json:"items"`
 }
+
+// +kubebuilder:validation:Enum=server;client;metrics-exporter
+type PgpoolCertificateAlias string
+
+const (
+	PgpoolServerCert          PgpoolCertificateAlias = "server"
+	PgpoolClientCert          PgpoolCertificateAlias = "client"
+	PgpoolMetricsExporterCert PgpoolCertificateAlias = "metrics-exporter"
+)
+
+// ref: https://www.postgresql.org/docs/13/libpq-ssl.html
+// +kubebuilder:validation:Enum=disable;allow;prefer;require;verify-ca;verify-full
+type PgpoolSSLMode string
+
+const (
+	// PgpoolSSLModeDisable represents `disable` sslMode. It ensures that the server does not use TLS/SSL.
+	PgpoolSSLModeDisable PgpoolSSLMode = "disable"
+
+	// PgpoolSSLModeAllow represents `allow` sslMode. 	I don't care about security,
+	// but I will pay the overhead of encryption if the server insists on it.
+	PgpoolSSLModeAllow PgpoolSSLMode = "allow"
+
+	// PgpoolSSLModePrefer represents `preferSSL` sslMode.
+	// I don't care about encryption, but I wish to pay the overhead of encryption if the server supports it.
+	PgpoolSSLModePrefer PgpoolSSLMode = "prefer"
+
+	// PgpoolSSLModeRequire represents `requiteSSL` sslmode. I want my data to be encrypted, and I accept the overhead.
+	// I trust that the network will make sure I always connect to the server I want.
+	PgpoolSSLModeRequire PgpoolSSLMode = "require"
+
+	// PgpoolSSLModeVerifyCA represents `verify-ca` sslmode. I want my data encrypted, and I accept the overhead.
+	// I want to be sure that I connect to a server that I trust.
+	PgpoolSSLModeVerifyCA PgpoolSSLMode = "verify-ca"
+
+	// PgpoolSSLModeVerifyFull represents `verify-full` sslmode. I want my data encrypted, and I accept the overhead.
+	// I want to be sure that I connect to a server I trust, and that it's the one I specify.
+	PgpoolSSLModeVerifyFull PgpoolSSLMode = "verify-full"
+)
+
+// PgpoolClientAuthMode represents the ClientAuthMode of Pgpool clusters ( replicaset )
+// ref: https://www.postgresql.org/docs/12/auth-methods.html
+// +kubebuilder:validation:Enum=md5;scram;cert
+type PgpoolClientAuthMode string
+
+const (
+	// PgpoolClientAuthModeMD5 uses a custom less secure challenge-response mechanism.
+	// It prevents password sniffing and avoids storing passwords on the server in plain text but provides no protection
+	// if an attacker manages to steal the password hash from the server.
+	// Also, the MD5 hash algorithm is nowadays no longer considered secure against determined attacks
+	PgpoolClientAuthModeMD5 PgpoolClientAuthMode = "md5"
+
+	// PgpoolClientAuthModeScram performs SCRAM-SHA-256 authentication, as described in RFC 7677.
+	// It is a challenge-response scheme that prevents password sniffing on untrusted connections
+	// and supports storing passwords on the server in a cryptographically hashed form that is thought to be secure.
+	// This is the most secure of the currently provided methods, but it is not supported by older client libraries.
+	PgpoolClientAuthModeScram PgpoolClientAuthMode = "scram"
+
+	// PgpoolClientAuthModeCert represents `cert clientcert=1` auth mode where client need to provide cert and private key for authentication.
+	// When server is config with this auth method. Client can't connect with pgpool server with password. They need
+	// to Send the client cert and client key certificate for authentication.
+	PgpoolClientAuthModeCert PgpoolClientAuthMode = "cert"
+)