Release PolarDB-X Operator v1.3.0-alpha.1

.gitignore

@@ -355,6 +355,7 @@ flycheck_*.el
 .idea/**/usage.statistics.xml
 .idea/**/dictionaries
 .idea/**/shelf
+.idea
 
 # Generated files
 .idea/**/contentModel.xml

CHANGELOG/README.md

@@ -1,16 +1,35 @@
 # Changelog
 
+## 2022-10-26
+
+Release v1.3.0-alpha.1
+
++ Enhancement & New Features
+  + Support backup and restore.
+  + Support multiple architecture: linux/amd64, linux/arm64.
+  + Support log(audit, error) collection.
+  + Support read only cluster.
+  + Support parameter template and configuration.
+  + Support XStore node rebuild.
+  + Support splitting dn data and log into different directories.
+  + Support archiving cold data into OSS and NFS.
++ Bug Fix
+  + Fix `mycnfOverwrite` not working issue.
+  + Fix issue of cn topology rules not working in host network mode.
+  + Fix XStore node not ready when ip changed in container network mode.
+
 ## 2022-06-07
 
 Release v1.2.2.
 
 This is a bugfix release.
-+ Fix DN scale up/down and upgrade issue.
-+ Fix grafana memory usage panel when deployed on minikube.
-+ Create xstore of DN in parallel.
-+ Fix node selector issue for polardbx-monitor.
-+ Disable service links in pod spec.
-+ Fix overriding my.cnf passing to the script without section.
+
+Fix DN scale up/down and upgrade issue.
+Fix grafana memory usage panel when deployed on minikube.
+Create xstore of DN in parallel.
+Fix node selector issue for polardbx-monitor.
+Disable service links in pod spec.
+Fix overriding my.cnf passing to the script without section.
 
 ## 2022-04-11
 

README.md

@@ -7,15 +7,15 @@ tasks.
 
 ## Quick Start
 
-Follow the [ [Quick Start](./docs/en/deploy/quick-start.md) / [快速开始](https://github.com/ApsaraDB/polardb-x-operator-docs/blob/main/deployment/0-quickstart.md) ] guide to start a testing Kubernetes cluster and play with PolarDB-X Operator.
+Follow the [[Quick Start](./docs/en/deploy/quick-start.md) / [快速开始](https://doc.polardbx.com/quickstart/topics/quickstart-k8s.html)] guide to start a testing Kubernetes cluster and play with PolarDB-X Operator.
 
 ## Documentation
 
 **NOTE: v1.2.0 introduces a breaking change to support the new xpaxos galaxyengine. Please refer to the [CHANGELOG](./CHANGELOG.md) for upgrade guidance.**
 
 Refer to the documentations for more details, such as CRD definitions and operation guides.
 
-+ [简体中文](https://github.com/ApsaraDB/polardb-x-operator-docs)
++ [简体中文](https://doc.polardbx.com/quickstart/topics/quickstart-k8s.html)
 + [English](docs/en/index.md)
 
 ## Roadmap
@@ -33,14 +33,17 @@ Implemented:
 + ✅ Automated Data Rebalance
 + ✅ Automated Failover
 + ✅ Security and TLS
-+ ✅ Multiple Architecture Support
-  + linux/amd64, linux/arm64
++ ✅ Multiple Architecture Support: linux/amd64, linux/arm64
++ ✅ Backup and Restore
++ ✅ Log(Audit, Error) Collection
++ ✅ Readonly instance
++ ✅ Parameter Template and Configuration
++ ✅ XStore Node Rebuild
 
 Working in progress:
 
-+ [T0] Backup and Restore 
-+ [T1] Backup Plan
-+ [T1] Configuration (DN) 
++ [T0] Backup Plan
++ [T1] Point in Time Recovery
 + [T2] Dashboard
 + ...
 

api/v1/backup_storage_provider.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2021 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+// BackupStorageProvider defines the configuration of storage for storing backup files.
+type BackupStorageProvider struct {
+	// StorageName defines the storage medium used to perform backup
+	StorageName BackupStorage `json:"storageName,omitempty"`
+
+	// Sink defines the storage configuration choose to perform backup
+	Sink string `json:"sink,omitempty"`
+	// TODO: Add Nas Provider
+}
+
+// BackupStorage defines the storage of backup
+type BackupStorage string
+
+const (
+	OSS  BackupStorage = "oss"
+	SFTP BackupStorage = "sftp"
+)

api/v1/polardbx/config.go

@@ -22,20 +22,35 @@ import (
 )
 
 type CNStaticConfig struct {
+	AttendHtap           bool                          `json:"AttendHtap,omitempty"`
 	EnableCoroutine      bool                          `json:"EnableCoroutine,omitempty"`
 	EnableReplicaRead    bool                          `json:"EnableReplicaRead,omitempty"`
 	EnableJvmRemoteDebug bool                          `json:"EnableJvmRemoteDebug,omitempty"`
 	ServerProperties     map[string]intstr.IntOrString `json:"ServerProperties,omitempty"`
+
+	// +kubebuilder:default=1
+	// +kubebuilder:validation:Enum=1;2;"1";"2";""
+
+	RPCProtocolVersion intstr.IntOrString `json:"RPCProtocolVersion,omitempty"`
 }
 
 type CNConfig struct {
 	Dynamic map[string]intstr.IntOrString `json:"dynamic,omitempty"`
-	Static  *CNStaticConfig               `json:"static,omitempty"`
+
+	// +kubebuilder:default={EnableCoroutine: true}
+
+	Static         *CNStaticConfig `json:"static,omitempty"`
+	EnableAuditLog bool            `json:"enableAuditLog,omitempty"`
+
+	// ColdDataFileStorage defines the file storage used to store cold data
+	ColdDataFileStorage []FileStorageInfo `json:"coldDataFileStorage,omitempty"`
 }
 
 type DNConfig struct {
-	MycnfOverwrite   string          `json:"mycnfOverwrite,omitempty"`
-	LogPurgeInterval metav1.Duration `json:"logPurgeInterval,omitempty"`
+	MycnfOverwrite    string          `json:"mycnfOverwrite,omitempty"`
+	LogPurgeInterval  metav1.Duration `json:"logPurgeInterval,omitempty"`
+	EnableAuditLog    bool            `json:"enableAuditLog,omitempty"`
+	LogDataSeparation bool            `json:"logDataSeparation,omitempty"`
 }
 
 type Config struct {

api/v1/polardbx/file_storage.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2021 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package polardbx
+
+import "strings"
+
+type FileStorageInfo struct {
+	// +kubebuilder:validation:Pattern=`^(?i)innodb|mrg_myisam|blackhole|myisam|csv|archive|performance_schema|federated|local_disk|s3|oss$`
+	// +kubebuilder:validation:Required
+
+	// Engine describes the engine type of file storage
+	Engine string `json:"engine,omitempty"`
+}
+
+type EngineType string
+
+const (
+	EngineTypeInnodb            EngineType = "INNODB"
+	EngineTypeMrgMyisam         EngineType = "MRG_MYISAM"
+	EngineTypeBlackhole         EngineType = "BLACKHOLE"
+	EngineTypeMyisam            EngineType = "MYISAM"
+	EngineTypeCsv               EngineType = "CSV"
+	EngineTypeArchive           EngineType = "ARCHIVE"
+	EngineTypePerformanceSchema EngineType = "PERFORMANCE_SCHEMA"
+	EngineTypeFederated         EngineType = "FEDERATED"
+	EngineTypeLocalDisk         EngineType = "LOCAL_DISK"
+	EngineTypeS3                EngineType = "S3"
+	EngineTypeOss               EngineType = "OSS"
+)
+
+func (info *FileStorageInfo) GetEngineType() EngineType {
+	return EngineType(strings.ToUpper(info.Engine))
+}
+
+func (info *FileStorageInfo) CheckEngineExists(existedFileStorages []FileStorageInfo) bool {
+	for _, existedFileStorage := range existedFileStorages {
+		if info.GetEngineType() == existedFileStorage.GetEngineType() {
+			return true
+		}
+	}
+	return false
+}

api/v1/polardbx/parameter.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2022 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package polardbx
+
+const (
+	RolingRestart = "rollingRestart"
+	Restart       = "restart"
+)
+
+type ParameterTemplate struct {
+	// parameter template
+	// +optional
+	Name string `json:"name,omitempty"`
+}
+
+type RestartingPods struct {
+	LastDeletedPod string   `json:"lastDeletedPod,omitempty"`
+	ToDeletePod    []string `json:"toDeletePod,omitempty"`
+}

api/v1/polardbx/readonly.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2021 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package polardbx
+
+import "k8s.io/apimachinery/pkg/util/intstr"
+
+type ReadonlyParam struct {
+	CnReplicas  int                           `json:"cnReplicas,omitempty"`
+	Name        string                        `json:"name,omitempty"`
+	ExtraParams map[string]intstr.IntOrString `json:"extraParams,omitempty"`
+}

api/v1/polardbx/restore.go

@@ -17,5 +17,26 @@ limitations under the License.
 package polardbx
 
 type RestoreSpec struct {
-	// TODO
+	//BackupSet defines the source of backup set
+	BackupSet string `json:"backupset,omitempty"`
+	// From defines the source information, either backup sets, snapshot or an running cluster.
+	From PolarDBXRestoreFrom `json:"from,omitempty"`
+
+	// Time defines the specified time of the restored data, in the format of 'yyyy-MM-dd HH:mm:ss'. Required.
+	Time string `json:"time,omitempty"`
+
+	// TimeZone defines the specified time zone of the restore time. Default is the location of current cluster.
+	// +optional
+	TimeZone string `json:"timezone,omitempty"`
+}
+
+// PolarDBXRestoreFrom defines the source information of the restored cluster.
+type PolarDBXRestoreFrom struct {
+	// PolarBDXName defines the the polardbx name that this polardbx is restored from. Optional.
+	// +optional
+	PolarBDXName string `json:"clusterName,omitempty"`
+
+	// BackupSelector defines the selector for the backups to be selected. Optional.
+	// +optional
+	BackupSelector map[string]string `json:"backupSelector,omitempty"`
 }

api/v1/polardbx/status.go

@@ -28,16 +28,17 @@ type Phase string
 
 // Valid phases.
 const (
-	PhaseNew       Phase = ""
-	PhasePending   Phase = "Pending"
-	PhaseCreating  Phase = "Creating"
-	PhaseRunning   Phase = "Running"
-	PhaseLocked    Phase = "Locked"
-	PhaseUpgrading Phase = "Upgrading"
-	PhaseRestoring Phase = "Restoring"
-	PhaseDeleting  Phase = "Deleting"
-	PhaseFailed    Phase = "Failed"
-	PhaseUnknown   Phase = "Unknown"
+	PhaseNew        Phase = ""
+	PhasePending    Phase = "Pending"
+	PhaseCreating   Phase = "Creating"
+	PhaseRunning    Phase = "Running"
+	PhaseLocked     Phase = "Locked"
+	PhaseUpgrading  Phase = "Upgrading"
+	PhaseRestoring  Phase = "Restoring"
+	PhaseDeleting   Phase = "Deleting"
+	PhaseFailed     Phase = "Failed"
+	PhaseRestarting Phase = "Restarting"
+	PhaseUnknown    Phase = "Unknown"
 )
 
 // Stage defines the operating stage of the cluster.
@@ -144,7 +145,7 @@ type ClusterReplicasStatus struct {
 	GMS ReplicasStatus `json:"gms,omitempty"`
 
 	// CN defines the replica status for CN.
-	CN ReplicasStatus `json:"cn,omitempty"`
+	CN *ReplicasStatus `json:"cn,omitempty"`
 
 	// DN defines the replica status for DN.
 	DN ReplicasStatus `json:"dn,omitempty"`
@@ -161,3 +162,12 @@ const (
 	MonitorStatusMonitoring MonitorStatus = "Monitoring"
 	MonitorStatusUpdating   MonitorStatus = "Updating"
 )
+
+type ParameterPhase string
+
+const (
+	ParameterStatusNew       ParameterPhase = ""
+	ParameterStatusCreating  ParameterPhase = "Creating"
+	ParameterStatusRunning   ParameterPhase = "Running"
+	ParameterStatusModifying ParameterPhase = "Modifying"
+)

api/v1/polardbx/topology.go

@@ -178,7 +178,7 @@ type TopologyNodeCN struct {
 	// +kubebuilder:default=2
 	// +kubebuilder:validation:Minimum=0
 
-	Replicas int32 `json:"replicas,omitempty"`
+	Replicas *int32 `json:"replicas,omitempty"`
 
 	// +kubebuilder:default={resources: {limits: {cpu: 4, memory: "8Gi"}}}
 
@@ -191,6 +191,10 @@ type TopologyNodeCDC struct {
 
 	Replicas int32 `json:"replicas,omitempty"`
 
+	// +kubebuilder:validation:Minimum=0
+
+	XReplicas int32 `json:"xReplicas,omitempty"`
+
 	// +kubebuilder:default={resources: {limits: {cpu: 4, memory: "8Gi"}}}
 
 	Template CDCTemplate `json:"template,omitempty"`
@@ -200,7 +204,6 @@ type TopologyNodes struct {
 	GMS TopologyNodeGMS `json:"gms,omitempty"`
 
 	// +kubebuilder:default={replicas:2,template:{resources:{limits:{cpu:4,memory:"8Gi"}}}}
-
 	CN TopologyNodeCN `json:"cn,omitempty"`
 
 	// +kubebuilder:default={replicas:2,template:{hostNetwork:true,resources:{limits:{cpu:4,memory:"8Gi"}}}}