From 25720c09521560f1f9d7ad1b5a3d785dee2bcf2b Mon Sep 17 00:00:00 2001 From: Ye Cao Date: Tue, 22 Aug 2023 10:17:50 +0800 Subject: [PATCH 1/4] Implement the vineyard csi driver and integrate it with kubeflow examples. * Add a new CRD CSIDriver to manage the deployment and status of the vineyard csi driver. * Add the CSIDriver benchmark test and relevant doc. * Introduce the kubernetes csi test to enhancing the robustness of the vineyard CSI Driver. * Generate the doc for CSIDriver CRD. Signed-off-by: Ye Cao --- go.work | 2 +- go/vineyard/go.mod | 2 +- k8s/.gitignore | 3 +- k8s/Dockerfile | 7 +- k8s/Makefile | 33 ++- k8s/PROJECT | 12 + k8s/apis/k8s/v1alpha1/README.md | 91 ++++++ k8s/apis/k8s/v1alpha1/csidriver_types.go | 144 ++++++++++ k8s/apis/k8s/v1alpha1/csidriver_webhook.go | 69 +++++ k8s/apis/k8s/v1alpha1/sidecar_webhook.go | 3 +- k8s/apis/k8s/v1alpha1/webhook_test.go | 6 + .../k8s/v1alpha1/zz_generated.deepcopy.go | 89 ++++++ k8s/cmd/commands/csi/csi.go | 50 ++++ k8s/cmd/commands/csi/csi_test.go | 67 +++++ k8s/cmd/commands/csi/pkg/controller.go | 269 ++++++++++++++++++ k8s/cmd/commands/csi/pkg/driver.go | 114 ++++++++ k8s/cmd/commands/csi/pkg/identity.go | 70 +++++ k8s/cmd/commands/csi/pkg/node.go | 183 ++++++++++++ k8s/cmd/commands/csi/pkg/state.go | 128 +++++++++ k8s/cmd/commands/flags/csi_flags.go | 38 +++ k8s/cmd/commands/manager/manager.go | 13 +- k8s/cmd/main.go | 2 + .../crd/bases/k8s.v6d.io_csidrivers.yaml | 94 ++++++ k8s/config/crd/kustomization.yaml | 8 +- .../patches/cainjection_in_csidrivers.yaml | 7 + ...vers.yaml => cainjection_in_recovers.yaml} | 0 .../crd/patches/webhook_in_csidrivers.yaml | 16 ++ ...recovers.yaml => webhook_in_recovers.yaml} | 0 k8s/config/manager/manager.yaml | 1 + .../rbac/k8s_csidriver_editor_role.yaml | 24 ++ .../rbac/k8s_csidriver_viewer_role.yaml | 20 ++ k8s/config/rbac/manager_account.yaml | 6 - k8s/config/rbac/role.yaml | 134 +++++++++ .../samples/k8s_v1alpha1_csidriver.yaml | 10 + k8s/config/webhook/manifests.yaml | 20 ++ k8s/controllers/k8s/csidriver_controller.go | 219 ++++++++++++++ k8s/csitest.Dockerfile | 25 ++ k8s/examples/vineyard-csidriver/Dockerfile | 12 + k8s/examples/vineyard-csidriver/Makefile | 22 ++ k8s/examples/vineyard-csidriver/README.md | 205 +++++++++++++ .../pipeline-with-vineyard.py | 82 ++++++ .../pipeline-with-vineyard.yaml | 264 +++++++++++++++++ k8s/examples/vineyard-csidriver/pipeline.py | 45 +++ k8s/examples/vineyard-csidriver/pipeline.yaml | 84 ++++++ .../vineyard-csidriver/prepare-data.yaml | 55 ++++ .../prepare-data/prepare-data.py | 68 +++++ .../preprocess/preprocess.py | 85 ++++++ k8s/examples/vineyard-csidriver/rbac.yaml | 30 ++ k8s/examples/vineyard-csidriver/test/test.py | 42 +++ .../vineyard-csidriver/train/train.py | 40 +++ k8s/go.mod | 12 +- k8s/go.sum | 15 +- k8s/pkg/schedulers/scheduling_strategy.go | 1 + k8s/pkg/templates/csidriver/daemonset.yaml | 110 +++++++ k8s/pkg/templates/csidriver/deployment.yaml | 104 +++++++ k8s/pkg/templates/csidriver/storageclass.yaml | 10 + k8s/pkg/templates/template.go | 2 +- k8s/test/e2e/Makefile | 3 +- 58 files changed, 3242 insertions(+), 28 deletions(-) create mode 100644 k8s/apis/k8s/v1alpha1/csidriver_types.go create mode 100644 k8s/apis/k8s/v1alpha1/csidriver_webhook.go create mode 100644 k8s/cmd/commands/csi/csi.go create mode 100644 k8s/cmd/commands/csi/csi_test.go create mode 100644 k8s/cmd/commands/csi/pkg/controller.go create mode 100644 k8s/cmd/commands/csi/pkg/driver.go create mode 100644 k8s/cmd/commands/csi/pkg/identity.go create mode 100644 k8s/cmd/commands/csi/pkg/node.go create mode 100644 k8s/cmd/commands/csi/pkg/state.go create mode 100644 k8s/cmd/commands/flags/csi_flags.go create mode 100644 k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml create mode 100644 k8s/config/crd/patches/cainjection_in_csidrivers.yaml rename k8s/config/crd/patches/{cainjection_in_k8s_recovers.yaml => cainjection_in_recovers.yaml} (100%) create mode 100644 k8s/config/crd/patches/webhook_in_csidrivers.yaml rename k8s/config/crd/patches/{webhook_in_k8s_recovers.yaml => webhook_in_recovers.yaml} (100%) create mode 100644 k8s/config/rbac/k8s_csidriver_editor_role.yaml create mode 100644 k8s/config/rbac/k8s_csidriver_viewer_role.yaml create mode 100644 k8s/config/samples/k8s_v1alpha1_csidriver.yaml create mode 100644 k8s/controllers/k8s/csidriver_controller.go create mode 100644 k8s/csitest.Dockerfile create mode 100644 k8s/examples/vineyard-csidriver/Dockerfile create mode 100644 k8s/examples/vineyard-csidriver/Makefile create mode 100644 k8s/examples/vineyard-csidriver/README.md create mode 100644 k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py create mode 100644 k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml create mode 100644 k8s/examples/vineyard-csidriver/pipeline.py create mode 100644 k8s/examples/vineyard-csidriver/pipeline.yaml create mode 100644 k8s/examples/vineyard-csidriver/prepare-data.yaml create mode 100644 k8s/examples/vineyard-csidriver/prepare-data/prepare-data.py create mode 100644 k8s/examples/vineyard-csidriver/preprocess/preprocess.py create mode 100644 k8s/examples/vineyard-csidriver/rbac.yaml create mode 100644 k8s/examples/vineyard-csidriver/test/test.py create mode 100644 k8s/examples/vineyard-csidriver/train/train.py create mode 100644 k8s/pkg/templates/csidriver/daemonset.yaml create mode 100644 k8s/pkg/templates/csidriver/deployment.yaml create mode 100644 k8s/pkg/templates/csidriver/storageclass.yaml diff --git a/go.work b/go.work index 6931331d..cec3d134 100644 --- a/go.work +++ b/go.work @@ -2,4 +2,4 @@ go 1.19 use ./go/vineyard -use ./k8s +use ./k8s \ No newline at end of file diff --git a/go/vineyard/go.mod b/go/vineyard/go.mod index 32b4d330..725a4cff 100644 --- a/go/vineyard/go.mod +++ b/go/vineyard/go.mod @@ -1,6 +1,6 @@ module github.com/v6d-io/v6d/go/vineyard -go 1.18 +go 1.19 require ( github.com/apache/arrow/go/v11 v11.0.0 diff --git a/k8s/.gitignore b/k8s/.gitignore index b84dbcc3..20e4a3d2 100644 --- a/k8s/.gitignore +++ b/k8s/.gitignore @@ -25,5 +25,4 @@ vineyardctl *~ # vendor, used for code-generate only -/vendor/ - +/vendor/ \ No newline at end of file diff --git a/k8s/Dockerfile b/k8s/Dockerfile index ac27a89f..c0438492 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -1,3 +1,4 @@ +ARG BASE_IMAGE=gcr.io/distroless/static:nonroot # Build the manager binary FROM golang:1.19 as builder @@ -21,14 +22,12 @@ RUN go mod download RUN go build -a -o vineyardctl cmd/main.go && \ strip vineyardctl || true + # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM ${BASE_IMAGE} WORKDIR / COPY k8s/config/scheduler/config.yaml /etc/kubernetes/scheduler.yaml COPY --from=builder /workspace/k8s/vineyardctl /vineyardctl -USER nonroot:nonroot - -ENTRYPOINT ["/vineyardctl", "manager"] diff --git a/k8s/Makefile b/k8s/Makefile index 86cb596d..be8a06a2 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -29,6 +29,8 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) # Image URL to use all building/pushing image targets REGISTRY := vineyardcloudnative IMG ?= $(REGISTRY)/vineyard-operator:$(VERSION) +VINEYARD_CSI_IMAGE ?= $(REGISTRY)/vineyard-csi-driver:$(VERSION) +VINEYARD_CSI_TEST_IMAGE ?= $(REGISTRY)/vineyard-csi-driver-test:$(VERSION) temp=$(shell mktemp -d) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -223,11 +225,38 @@ vendor: docker-build: cd .. && \ if docker build --help | grep -q load; then \ - docker build --load -f k8s/Dockerfile . -t $(IMG); \ + docker build --load -f k8s/Dockerfile . \ + -t $(IMG); \ else \ - docker build -f k8s/Dockerfile . -t $(IMG); \ + docker build -f k8s/Dockerfile . \ + -t $(IMG); \ fi +.PHONY: csidriver +csidriver: + cd .. && \ + if docker build --help | grep -q load; then \ + docker build --load -f k8s/Dockerfile . \ + --build-arg BASE_IMAGE=golang:1.19-buster \ + -t $(VINEYARD_CSI_IMAGE); \ + else \ + docker build -f k8s/Dockerfile . \ + --build-arg BASE_IMAGE=golang:1.19-buster \ + -t $(VINEYARD_CSI_IMAGE); \ + fi + +.PHONY: csidriver-test +csidriver-test: + cd .. && \ + if docker build --help | grep -q load; then \ + docker build --load -f k8s/csitest.Dockerfile . \ + -t $(VINEYARD_CSI_TEST_IMAGE); \ + else \ + docker build -f k8s/csitest.Dockerfile . \ + -t $(VINEYARD_CSI_TEST_IMAGE); \ + fi && \ + docker run --rm -it --privileged=true $(VINEYARD_CSI_TEST_IMAGE) + docker-build-push-multi-arch: cd .. && \ docker buildx build -f k8s/Dockerfile . -t $(IMG) --platform linux/amd64,linux/arm64 --push diff --git a/k8s/PROJECT b/k8s/PROJECT index f1a47f28..174effef 100644 --- a/k8s/PROJECT +++ b/k8s/PROJECT @@ -96,4 +96,16 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + controller: true + domain: v6d.io + group: k8s + kind: CSIDriver + path: github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1 + version: v1alpha1 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 version: "3" diff --git a/k8s/apis/k8s/v1alpha1/README.md b/k8s/apis/k8s/v1alpha1/README.md index 0fc51655..329a2e4d 100644 --- a/k8s/apis/k8s/v1alpha1/README.md +++ b/k8s/apis/k8s/v1alpha1/README.md @@ -11,6 +11,8 @@ Package v1alpha1 contains API Schema definitions for the k8s v1alpha1 API group ### Resource Types - [Backup](#backup) - [BackupList](#backuplist) +- [CSIDriver](#csidriver) +- [CSIDriverList](#csidriverlist) - [GlobalObject](#globalobject) - [GlobalObjectList](#globalobjectlist) - [LocalObject](#localobject) @@ -80,6 +82,80 @@ _Appears in:_ +#### CSIDriver + + + +CSIDriver is the Schema for the csidrivers API + +_Appears in:_ +- [CSIDriverList](#csidriverlist) + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `k8s.v6d.io/v1alpha1` +| `kind` _string_ | `CSIDriver` +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CSIDriverSpec](#csidriverspec)_ | | + + +#### CSIDriverList + + + +CSIDriverList contains a list of CSIDriver + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `k8s.v6d.io/v1alpha1` +| `kind` _string_ | `CSIDriverList` +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `items` _[CSIDriver](#csidriver) array_ | | + + +#### CSIDriverSpec + + + +CSIDriverSpec defines the desired state of CSIDriver + +_Appears in:_ +- [CSIDriver](#csidriver) + +| Field | Description | +| --- | --- | +| `image` _string_ | Image is the name of the csi driver image | +| `imagePullPolicy` _string_ | ImagePullPolicy is the image pull policy of the csi driver | +| `storageClassName` _string_ | StorageClassName is the name of the storage class | +| `volumeBindingMode` _string_ | VolumeBindingMode is the volume binding mode of the storage class | +| `sidecar` _[CSISidecar](#csisidecar)_ | Sidecar is the configuration for the CSI sidecar container nolint: lll | +| `clusters` _[VineyardClusters](#vineyardclusters) array_ | Clusters are the list of vineyard clusters | +| `enableToleration` _boolean_ | EnableToleration is the flag to enable toleration for the csi driver | + + + + +#### CSISidecar + + + +CSISidecar holds the configuration for the CSI sidecar container + +_Appears in:_ +- [CSIDriverSpec](#csidriverspec) + +| Field | Description | +| --- | --- | +| `provisionerImage` _string_ | ProvisionerImage is the image of the provisioner sidecar | +| `attacherImage` _string_ | AttacherImage is the image of the attacher sidecar | +| `nodeRegistrarImage` _string_ | NodeRegistrarImage is the image of the node registrar sidecar | +| `livenessProbeImage` _string_ | LivenessProbeImage is the image of the liveness probe sidecar | +| `imagePullPolicy` _string_ | ImagePullPolicy is the image pull policy of all sidecar containers | +| `enableTopology` _boolean_ | EnableTopology is the flag to enable topology for the csi driver | + + #### GlobalObject @@ -419,6 +495,21 @@ _Appears in:_ | `persistentVolumeClaimSpec` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#persistentvolumeclaimspec-v1-core)_ | the PersistentVolumeClaimSpec of the spill file | +#### VineyardClusters + + + +VineyardClusters contains the list of vineyard clusters + +_Appears in:_ +- [CSIDriverSpec](#csidriverspec) + +| Field | Description | +| --- | --- | +| `namespace` _string_ | Namespace is the namespace of the vineyard cluster | +| `name` _string_ | Name is the name of the vineyard deployment | + + #### VineyardConfig diff --git a/k8s/apis/k8s/v1alpha1/csidriver_types.go b/k8s/apis/k8s/v1alpha1/csidriver_types.go new file mode 100644 index 00000000..b3027ef5 --- /dev/null +++ b/k8s/apis/k8s/v1alpha1/csidriver_types.go @@ -0,0 +1,144 @@ +/** Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CSISidecar holds the configuration for the CSI sidecar container +type CSISidecar struct { + // ProvisionerImage is the image of the provisioner sidecar + // +kubebuilder:validation:Required + // +kubebuilder:default:="registry.k8s.io/sig-storage/csi-provisioner:v3.3.0" + ProvisionerImage string `json:"provisionerImage,omitempty"` + + // AttacherImage is the image of the attacher sidecar + // +kubebuilder:validation:Required + // +kubebuilder:default:="registry.k8s.io/sig-storage/csi-attacher:v4.0.0" + AttacherImage string `json:"attacherImage,omitempty"` + + // NodeRegistrarImage is the image of the node registrar sidecar + // +kubebuilder:validation:Required + // +kubebuilder:default:="registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0" + NodeRegistrarImage string `json:"nodeRegistrarImage,omitempty"` + + // LivenessProbeImage is the image of the liveness probe sidecar + // +kubebuilder:validation:Required + // +kubebuilder:default:="registry.k8s.io/sig-storage/livenessprobe:v2.8.0" + LivenessProbeImage string `json:"livenessProbeImage,omitempty"` + + // ImagePullPolicy is the image pull policy of all sidecar containers + // +kubebuilder:validation:Required + // +kubebuilder:default:="Always" + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + + // EnableTopology is the flag to enable topology for the csi driver + // +kubebuilder:validation:Required + // +kubebuilder:default:=false + EnableTopology bool `json:"enableTopology,omitempty"` +} + +// VineyardClusters contains the list of vineyard clusters +type VineyardClusters struct { + // Namespace is the namespace of the vineyard cluster + // +kubebuilder:validation:Required + // +kubebuilder:default:="" + Namespace string `json:"namespace,omitempty"` + + // Name is the name of the vineyard deployment + // +kubebuilder:validation:Required + // +kubebuilder:default:="" + Name string `json:"name,omitempty"` +} + +// CSIDriverSpec defines the desired state of CSIDriver +type CSIDriverSpec struct { + // Image is the name of the csi driver image + // +kubebuilder:validation:Required + // +kubebuilder:default:="vineyardcloudnative/vineyard-csi-driver" + Image string `json:"image,omitempty"` + + // ImagePullPolicy is the image pull policy of the csi driver + // +kubebuilder:validation:Required + // +kubebuilder:default:="IfNotPresent" + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + + // StorageClassName is the name of the storage class + // +kubebuilder:validation:Required + // +kubebuilder:default:="vineyard-csi" + StorageClassName string `json:"storageClassName,omitempty"` + + // VolumeBindingMode is the volume binding mode of the storage class + // +kubebuilder:validation:Required + // +kubebuilder:default:="WaitForFirstConsumer" + VolumeBindingMode string `json:"volumeBindingMode,omitempty"` + + // Sidecar is the configuration for the CSI sidecar container + // +kubebuilder:validation:Required + //nolint: lll + // +kubebuilder:default:={provisionerImage: "registry.k8s.io/sig-storage/csi-provisioner:v3.3.0", attacherImage: "registry.k8s.io/sig-storage/csi-attacher:v4.0.0", nodeRegistrarImage: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0", livenessProbeImage: "registry.k8s.io/sig-storage/livenessprobe:v2.8.0", imagePullPolicy: "Always", enableTopology: false} + Sidecar CSISidecar `json:"sidecar,omitempty"` + + // Clusters are the list of vineyard clusters + // +kubebuilder:validation:Required + // +kubebuilder:default:={} + Clusters []VineyardClusters `json:"clusters,omitempty"` + + // EnableToleration is the flag to enable toleration for the csi driver + // +kubebuilder:validation:Required + // +kubebuilder:default:=false + EnableToleration bool `json:"enableToleration,omitempty"` +} + +// CSIDriverStatus defines the observed state of CSIDriver +type CSIDriverStatus struct { + // State is the state of the csi driver + State string `json:"state,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster + +// CSIDriver is the Schema for the csidrivers API +type CSIDriver struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CSIDriverSpec `json:"spec,omitempty"` + Status CSIDriverStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CSIDriverList contains a list of CSIDriver +type CSIDriverList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CSIDriver `json:"items"` +} + +const ( + // CSIDriverRunning is the running state of the csi driver + CSIDriverRunning = "running" + // CSIDriverPending is the pending state of the csi driver + CSIDriverPending = "pending" +) + +func init() { + SchemeBuilder.Register(&CSIDriver{}, &CSIDriverList{}) +} diff --git a/k8s/apis/k8s/v1alpha1/csidriver_webhook.go b/k8s/apis/k8s/v1alpha1/csidriver_webhook.go new file mode 100644 index 00000000..ada36263 --- /dev/null +++ b/k8s/apis/k8s/v1alpha1/csidriver_webhook.go @@ -0,0 +1,69 @@ +/** Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/v6d-io/v6d/k8s/pkg/log" +) + +// log is for logging in this package. +var csidriverlog = log.WithName("csidriver-resource") + +func (r *CSIDriver) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// nolint: lll +// +kubebuilder:webhook:path=/mutate-k8s-v6d-io-v1alpha1-csidriver,mutating=true,failurePolicy=fail,sideEffects=None,groups=k8s.v6d.io,resources=csidrivers,verbs=create;update,versions=v1alpha1,name=mcsidriver.kb.io,admissionReviewVersions=v1 +var _ webhook.Defaulter = &CSIDriver{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *CSIDriver) Default() { + csidriverlog.Info("default", "name", r.Name) + +} + +//nolint: lll +//+kubebuilder:webhook:path=/validate-k8s-v6d-io-v1alpha1-csidriver,mutating=false,failurePolicy=fail,sideEffects=None,groups=k8s.v6d.io,resources=csidrivers,verbs=create;update,versions=v1alpha1,name=vcsidriver.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &CSIDriver{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *CSIDriver) ValidateCreate() error { + csidriverlog.Info("validate create", "name", r.Name) + + return nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *CSIDriver) ValidateUpdate(old runtime.Object) error { + csidriverlog.Info("validate update", "name", r.Name) + + return nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *CSIDriver) ValidateDelete() error { + csidriverlog.Info("validate delete", "name", r.Name) + + return nil +} diff --git a/k8s/apis/k8s/v1alpha1/sidecar_webhook.go b/k8s/apis/k8s/v1alpha1/sidecar_webhook.go index 154212be..4f6a7716 100644 --- a/k8s/apis/k8s/v1alpha1/sidecar_webhook.go +++ b/k8s/apis/k8s/v1alpha1/sidecar_webhook.go @@ -16,10 +16,11 @@ limitations under the License. package v1alpha1 import ( - "github.com/v6d-io/v6d/k8s/pkg/log" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/v6d-io/v6d/k8s/pkg/log" ) // log is for logging in this package. diff --git a/k8s/apis/k8s/v1alpha1/webhook_test.go b/k8s/apis/k8s/v1alpha1/webhook_test.go index cd9bb277..74b7027d 100644 --- a/k8s/apis/k8s/v1alpha1/webhook_test.go +++ b/k8s/apis/k8s/v1alpha1/webhook_test.go @@ -1,8 +1,11 @@ /** Copyright 2020-2023 Alibaba Group Holding Limited. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -107,6 +110,9 @@ func Test_webhook(t *testing.T) { err = (&Recover{}).SetupWebhookWithManager(mgr) assert.NoError(t, err) + err = (&CSIDriver{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:webhook go func() { diff --git a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go index 21b5c749..ce8e2277 100644 --- a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go +++ b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go @@ -122,6 +122,95 @@ func (in *BackupStatus) DeepCopy() *BackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIDriver) DeepCopyInto(out *CSIDriver) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriver. +func (in *CSIDriver) DeepCopy() *CSIDriver { + if in == nil { + return nil + } + out := new(CSIDriver) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CSIDriver) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIDriverList) DeepCopyInto(out *CSIDriverList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CSIDriver, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverList. +func (in *CSIDriverList) DeepCopy() *CSIDriverList { + if in == nil { + return nil + } + out := new(CSIDriverList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CSIDriverList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverSpec. +func (in *CSIDriverSpec) DeepCopy() *CSIDriverSpec { + if in == nil { + return nil + } + out := new(CSIDriverSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIDriverStatus) DeepCopyInto(out *CSIDriverStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverStatus. +func (in *CSIDriverStatus) DeepCopy() *CSIDriverStatus { + if in == nil { + return nil + } + out := new(CSIDriverStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalObject) DeepCopyInto(out *GlobalObject) { *out = *in diff --git a/k8s/cmd/commands/csi/csi.go b/k8s/cmd/commands/csi/csi.go new file mode 100644 index 00000000..0ff38be1 --- /dev/null +++ b/k8s/cmd/commands/csi/csi.go @@ -0,0 +1,50 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package csi contains the start command of vineyard csi driver +package csi + +import ( + "github.com/spf13/cobra" + + "github.com/v6d-io/v6d/k8s/cmd/commands/csi/pkg" + "github.com/v6d-io/v6d/k8s/cmd/commands/flags" + "github.com/v6d-io/v6d/k8s/cmd/commands/util" +) + +var csiExample = util.Examples(` + # start the csidriver with the specific endpoint and node id + vineyardctl csidriver --endpoint=unix:///csi/csi.sock --nodeid=csinode1`) + +// csiCmd starts the vineyard csi driver +var csiCmd = &cobra.Command{ + Use: "csi", + Short: "Start the vineyard csi driver", + Example: csiExample, + Run: func(cmd *cobra.Command, args []string) { + util.AssertNoArgs(cmd, args) + d := pkg.NewDriver(flags.NodeID, flags.Endpoint) + d.Run() + }, +} + +func NewCsiCmd() *cobra.Command { + return csiCmd +} + +func init() { + flags.ApplyCsiOpts(csiCmd) +} diff --git a/k8s/cmd/commands/csi/csi_test.go b/k8s/cmd/commands/csi/csi_test.go new file mode 100644 index 00000000..f0925f17 --- /dev/null +++ b/k8s/cmd/commands/csi/csi_test.go @@ -0,0 +1,67 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package csi contains the start command of vineyard csi driver +package csi + +import ( + "os" + "os/signal" + "path/filepath" + "syscall" + "testing" + + "github.com/kubernetes-csi/csi-test/v4/pkg/sanity" + "github.com/v6d-io/v6d/k8s/cmd/commands/csi/pkg" + "github.com/v6d-io/v6d/k8s/cmd/commands/flags" +) + +func TestVineyardCSIDriver(t *testing.T) { + // Setup the full driver and its environment + csiSocket := "/tmp/csi.sock" + csiEndpoint := "unix://" + csiSocket + if err := os.Remove(csiSocket); err != nil && !os.IsNotExist(err) { + t.Errorf("failed to remove socket file %s: %v", csiSocket, err) + os.Exit(1) + } + + flags.Endpoint = csiEndpoint + flags.NodeID = "test-node-id" + + vineyardSocket := filepath.Join(pkg.VineyardSocketPrefix, pkg.VineyardSocket) + if _, err := os.OpenFile(vineyardSocket, os.O_CREATE|os.O_RDONLY, 0666); err != nil { + t.Errorf("failed to open vineyard socket file %s: %v", vineyardSocket, err) + } + + // Create a channel to signal the goroutine to stop + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + go func() { + csiCmd.Run(csiCmd, []string{}) + }() + config := sanity.NewTestConfig() + config.Address = csiEndpoint + config.TargetPath = "/opt/target" + config.StagingPath = "/opt/staging" + + sanity.Test(t, config) + + // Wait for a stop signal + <-stop + + // Exit the program with a status code of 0 (success) + os.Exit(0) +} diff --git a/k8s/cmd/commands/csi/pkg/controller.go b/k8s/cmd/commands/csi/pkg/controller.go new file mode 100644 index 00000000..2fb94877 --- /dev/null +++ b/k8s/cmd/commands/csi/pkg/controller.go @@ -0,0 +1,269 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package pkg + +import ( + "context" + "sync" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/v6d-io/v6d/k8s/pkg/log" +) + +var ( + controllerCaps = []csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + } + VineyardSocketPrefix = "/var/run/vineyard-kubernetes" + VineyardSocket = "vineyard.sock" + + // VineyardNamespaceKey is the key for the vineyard name in the volume context + VineyardNamespaceKey = "k8s.v6d.io/vineyard/namespace" + + // VineyardNameKey is the key for the vineyard name in the volume context + VineyardNameKey = "k8s.v6d.io/vineyard/name" +) + +type VineyardCSI struct { + mutex sync.Mutex + state *VolumeStates + nodeID string +} + +func NewVineyardCSI(stateFilePath string, nodeID string) *VineyardCSI { + volumeStates, err := NewVolumeStates(stateFilePath) + if err != nil { + log.Fatalf(err, "failed to create volume states") + } + return &VineyardCSI{ + mutex: sync.Mutex{}, + state: volumeStates, + nodeID: nodeID, + } +} + +func (vc *VineyardCSI) ControllerGetCapabilities(ctx context.Context, + req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + log.Infof("ControllerGetCapabilities: called with args %+v", *req) + caps := []*csi.ControllerServiceCapability{} + for _, cap := range controllerCaps { + c := &csi.ControllerServiceCapability{ + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: cap, + }, + }, + } + caps = append(caps, c) + } + return &csi.ControllerGetCapabilitiesResponse{Capabilities: caps}, nil +} + +func (vc *VineyardCSI) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + log.Infof("CreateVolume: called with args %+v", *req) + // check arguments + if req.Name == "" { + return nil, status.Error(codes.InvalidArgument, "CreateVolume Name must be provided") + } + + if req.VolumeCapabilities == nil || len(req.VolumeCapabilities) == 0 { + return nil, status.Error(codes.InvalidArgument, "CreateVolume Volume capabilities must be provided") + } + vc.mutex.Lock() + defer vc.mutex.Unlock() + + capacity := req.GetCapacityRange().GetRequiredBytes() + // check whether the volume exists + if exVol, err := vc.state.GetVolumeByName(req.Name); err == nil { + if exVol.VolSize != capacity { + return nil, status.Errorf(codes.AlreadyExists, "Volume with the same name: %s but with different size already exist", req.GetName()) + } + + // volume already exists, return the volume + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: exVol.VolID, + VolumeContext: req.GetParameters(), + }, + }, nil + } + // create the uuid for the volume + uuid, err := uuid.NewUUID() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + volumeID := uuid.String() + log.Infof("create volumeID: %s", volumeID) + + volumeContext := req.GetParameters() + if len(volumeContext) == 0 { + volumeContext = make(map[string]string) + } + volumeContext["volume_id"] = volumeID + + // add volume to the state + volume := Volume{ + VolID: volumeID, + VolName: req.Name, + VolSize: capacity, + } + if err := vc.state.AddVolume(volume); err != nil { + return nil, err + } + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: volumeID, + VolumeContext: volumeContext, + }, + }, nil +} + +func (vc *VineyardCSI) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + log.Infof("DeleteVolume: called with args: %+v", *req) + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "DeleteVolume Volume ID must be provided") + } + + if err := vc.state.DeleteVolume(req.VolumeId); err != nil { + return nil, err + } + return &csi.DeleteVolumeResponse{}, nil +} + +func (vc *VineyardCSI) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { + log.Infof("ControllerGetVolume: called with args: %+v", *req) + return &csi.ControllerGetVolumeResponse{}, nil +} + +func (vc *VineyardCSI) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + log.Infof("ControllerPublishVolume: called with args %+v", *req) + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume ID must be provided") + } + + if req.NodeId == "" { + return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Node ID must be provided") + } + + if req.VolumeCapability == nil { + return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume capability must be provided") + } + + vc.mutex.Lock() + defer vc.mutex.Unlock() + + vol, err := vc.state.GetVolumeByID(req.GetVolumeId()) + if err != nil { + return nil, err + } + + if vol.Attched { + if req.GetReadonly() != vol.ReadOnlyAttach { + return nil, status.Error(codes.AlreadyExists, "Volume published but has incompatible readonly flag") + } + + return &csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{}, + }, nil + } + + vol.Attched = true + vol.ReadOnlyAttach = req.GetReadonly() + + if err := vc.state.AddVolume(vol); err != nil { + return nil, err + } + return &csi.ControllerPublishVolumeResponse{}, nil +} + +func (vc *VineyardCSI) ControllerUnpublishVolume(ctx context.Context, + req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + log.Infof("ControllerUnpublishVolume: called with args %+v", *req) + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume ID must be provided") + } + return &csi.ControllerUnpublishVolumeResponse{}, nil +} + +func (vc *VineyardCSI) ValidateVolumeCapabilities(ctx context.Context, + req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities Volume ID must be provided") + } + + if req.VolumeCapabilities == nil { + return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities Volume Capabilities must be provided") + } + vc.mutex.Lock() + defer vc.mutex.Unlock() + + if _, err := vc.state.GetVolumeByID(req.GetVolumeId()); err != nil { + return nil, err + } + + for _, cap := range req.GetVolumeCapabilities() { + if cap.GetMount() == nil && cap.GetBlock() == nil { + return nil, status.Error(codes.InvalidArgument, "cannot have both mount and block access type be undefined") + } + } + + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ + VolumeContext: req.GetVolumeContext(), + VolumeCapabilities: req.GetVolumeCapabilities(), + Parameters: req.GetParameters(), + }, + }, nil + +} + +func (vc *VineyardCSI) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +func (vc *VineyardCSI) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +func (vc *VineyardCSI) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + if req.GetName() == "" { + return nil, status.Error(codes.InvalidArgument, "CreateSnapshot Name must be provided") + } + + if req.GetSourceVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "CreateSnapshot Source Volume ID must be provided") + } + + return nil, status.Error(codes.Unimplemented, "") +} + +func (vc *VineyardCSI) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +func (vc *VineyardCSI) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +func (vc *VineyardCSI) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} diff --git a/k8s/cmd/commands/csi/pkg/driver.go b/k8s/cmd/commands/csi/pkg/driver.go new file mode 100644 index 00000000..1a2dfd8d --- /dev/null +++ b/k8s/cmd/commands/csi/pkg/driver.go @@ -0,0 +1,114 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package pkg + +import ( + "context" + "fmt" + "net" + "os" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" + "github.com/pkg/errors" + "google.golang.org/grpc" + + "github.com/v6d-io/v6d/k8s/cmd/commands/flags" + "github.com/v6d-io/v6d/k8s/pkg/log" +) + +type Driver struct { + nodeID string + endpoint string +} + +const ( + version = "0.1.0" + driverName = "csi.vineyard.v6d.io" +) + +func NewDriver(nodeID, endpoint string) *Driver { + log.Infof("Driver: %v version: %v", driverName, version) + + n := &Driver{ + nodeID: nodeID, + endpoint: endpoint, + } + + return n +} + +func (d *Driver) Run() { + + vineyardCSI := NewVineyardCSI(flags.StateFilePath, d.nodeID) + identity := NewIdentityServer() + + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor(logGRPC), + } + + srv := grpc.NewServer(opts...) + + csi.RegisterControllerServer(srv, vineyardCSI) + csi.RegisterIdentityServer(srv, identity) + csi.RegisterNodeServer(srv, vineyardCSI) + + proto, addr, err := ParseEndpoint(d.endpoint) + log.Infof("protocol: %s,addr: %s", proto, addr) + if err != nil { + log.Fatalf(err, "Invalid endpoint: %v", d.endpoint) + } + + if proto == "unix" { + addr = "/" + addr + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + log.Fatalf(err, "Failed to remove %s, error: %s", addr, err.Error()) + } + } + + listener, err := net.Listen(proto, addr) + if err != nil { + log.Fatalf(err, "Failed to listen") + } + + if err := srv.Serve(listener); err != nil { + log.Fatalf(err, "Failed to serve") + } +} + +func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + log.Infof("GRPC call: %s", info.FullMethod) + log.Infof("GRPC request: %s", protosanitizer.StripSecrets(req)) + + resp, err := handler(ctx, req) + if err != nil { + log.Errorf(err, "GRPC error: %s", err.Error()) + } else { + log.Infof("GRPC response: %s", protosanitizer.StripSecrets(resp)) + } + return resp, err +} + +func ParseEndpoint(ep string) (string, string, error) { + if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") { + s := strings.SplitN(ep, "://", 2) + if s[1] != "" { + return s[0], s[1], nil + } + } + return "", "", errors.Wrap(fmt.Errorf("invalid endpoint: %s", ep), "parse endpoint") +} diff --git a/k8s/cmd/commands/csi/pkg/identity.go b/k8s/cmd/commands/csi/pkg/identity.go new file mode 100644 index 00000000..dd565aca --- /dev/null +++ b/k8s/cmd/commands/csi/pkg/identity.go @@ -0,0 +1,70 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package pkg + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + + "github.com/v6d-io/v6d/k8s/pkg/log" +) + +type IdentityServer struct { +} + +func NewIdentityServer() *IdentityServer { + return &IdentityServer{} +} + +func (ids *IdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + log.Infof("GetPluginInfo: called with args %+v", *req) + + return &csi.GetPluginInfoResponse{ + Name: driverName, + VendorVersion: version, + }, nil +} + +func (ids *IdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + log.Infof("GetPluginCapabilities: called with args %+v", *req) + + resp := &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, + }, + }, + }, + }, + } + + return resp, nil +} + +func (ids *IdentityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { + log.Infof("Probe: called with args %+v", *req) + return &csi.ProbeResponse{}, nil +} diff --git a/k8s/cmd/commands/csi/pkg/node.go b/k8s/cmd/commands/csi/pkg/node.go new file mode 100644 index 00000000..0deaad05 --- /dev/null +++ b/k8s/cmd/commands/csi/pkg/node.go @@ -0,0 +1,183 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package pkg + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/mount-utils" + + "github.com/v6d-io/v6d/k8s/pkg/log" +) + +// MaxVolumesPerNode is the maximum number of volumes supported per node. +const MaxVolumesPerNode = 1000 + +func (vc *VineyardCSI) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + log.Infof("NodeStageVolume: called with args %+v", *req) + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume ID must be provided") + } + + if req.StagingTargetPath == "" { + return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Staging Target Path must be provided") + } + + if req.VolumeCapability == nil { + return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume Capability must be provided") + } + return &csi.NodeStageVolumeResponse{}, nil +} + +func (vc *VineyardCSI) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + log.Infof("NodeUnstageVolume: called with args %+v", *req) + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "NodeUnstageVolume Volume ID must be provided") + } + + if req.StagingTargetPath == "" { + return nil, status.Error(codes.InvalidArgument, "NodeUnstageVolume Staging Target Path must be provided") + } + + return &csi.NodeUnstageVolumeResponse{}, nil +} + +func (vc *VineyardCSI) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + log.Infof("NodePublishVolume: called with args %+v", *req) + + // Check arguments + if req.GetVolumeCapability() == nil { + return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") + } + if len(req.GetVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + if len(req.GetTargetPath()) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path missing in request") + } + log.Infof("args check success") + + volumeContext := req.VolumeContext + vineyardNamespace, ok := volumeContext[VineyardNamespaceKey] + if !ok { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("vineyard namespace %s not found", VineyardNamespaceKey)) + } + vineyardName, ok := volumeContext[VineyardNameKey] + if !ok { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("vineyard name %s not found", VineyardNameKey)) + } + + vineyardSocketDir := filepath.Join(VineyardSocketPrefix, vineyardNamespace, vineyardName) + socket := filepath.Join(vineyardSocketDir, VineyardSocket) + if _, err := os.Stat(socket); err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("vineyard socket %s not found: %v", socket, err)) + } + + // check if target directory exists, create if not + targetPath := req.GetTargetPath() + log.Infof("target path: %s", targetPath) + _, err := os.Stat(targetPath) + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("create directory path {%s} error: %v", targetPath, err)) + } + } else if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("get target path {%s} failed: %v", targetPath, err)) + } + + mounter := mount.New("") + options := []string{"bind"} + if err := mounter.Mount(vineyardSocketDir, targetPath, "", options); err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("mount %s to %s failed: %v", vineyardSocketDir, targetPath, err)) + } + log.Infof("mount %s to %s success", vineyardSocketDir, req.TargetPath) + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (vc *VineyardCSI) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + log.Infof("NodeUnpublishVolume: called with args %+v", *req) + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Volume ID must be provided") + } + + if req.TargetPath == "" { + return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Target Path must be provided") + } + + targetPath := req.GetTargetPath() + if err := mount.CleanupMountPoint(targetPath, mount.New(""), true); err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("unmount %s failed: %v", targetPath, err)) + } + log.Infof("unmount %s success", targetPath) + + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +func (vc *VineyardCSI) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + log.Infof("NodeGetInfo: called with args %+v", *req) + + return &csi.NodeGetInfoResponse{ + NodeId: vc.nodeID, + MaxVolumesPerNode: MaxVolumesPerNode, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + "kubernetes.io/hostname": vc.nodeID, + }, + }, + }, nil +} + +func (vc *VineyardCSI) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + log.Infof("NodeGetCapabilities: called with args %+v", *req) + + return &csi.NodeGetCapabilitiesResponse{ + Capabilities: []*csi.NodeServiceCapability{ + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }, + }, + }, nil +} + +func (vc *VineyardCSI) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats Volume ID must be provided") + } + return nil, status.Error(codes.Unimplemented, "") +} + +func (vc *VineyardCSI) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + if len(req.GetVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeExpandVolume volume ID not provided") + } + + if len(req.GetVolumePath()) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeExpandVolume volume path not provided") + } + return nil, status.Error(codes.Unimplemented, "") +} diff --git a/k8s/cmd/commands/csi/pkg/state.go b/k8s/cmd/commands/csi/pkg/state.go new file mode 100644 index 00000000..23ec6476 --- /dev/null +++ b/k8s/cmd/commands/csi/pkg/state.go @@ -0,0 +1,128 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkg + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type Volume struct { + VolName string + VolID string + VolSize int64 + VolPath string + NodeID string + ReadOnlyAttach bool + Attched bool +} + +type VolumeStates struct { + Volumes []Volume + StateFilePath string +} + +func NewVolumeStates(statefilePath string) (*VolumeStates, error) { + s := &VolumeStates{ + StateFilePath: statefilePath, + } + + return s, s.restore() +} + +func (s *VolumeStates) dump() error { + data, err := json.Marshal(&s.Volumes) + if err != nil { + return status.Errorf(codes.Internal, "error encoding volumes and snapshots: %v", err) + } + if err := os.WriteFile(s.StateFilePath, data, 0600); err != nil { + return status.Errorf(codes.Internal, "error writing state file: %v", err) + } + return nil +} + +func (s *VolumeStates) restore() error { + s.Volumes = nil + + data, err := os.ReadFile(s.StateFilePath) + switch { + case errors.Is(err, os.ErrNotExist): + // create the /csi directory if it does not exist + if err := os.MkdirAll(filepath.Dir(s.StateFilePath), 0750); err != nil { + return status.Errorf(codes.Internal, "error creating state file directory: %v", err) + } + // create the state file if it does not exist + if _, err := os.Create(s.StateFilePath); err != nil { + return status.Errorf(codes.Internal, "error creating state file: %v", err) + } + return nil + case err != nil: + return status.Errorf(codes.Internal, "error reading state file: %v", err) + } + if len(data) == 0 { + s.Volumes = []Volume{} + return nil + } + if err := json.Unmarshal(data, &s.Volumes); err != nil { + return status.Errorf(codes.Internal, "error encoding volumes and snapshots from state file %q: %v", s.StateFilePath, err) + } + return nil +} + +func (s *VolumeStates) GetVolumeByID(volID string) (Volume, error) { + for _, volume := range s.Volumes { + if volume.VolID == volID { + return volume, nil + } + } + return Volume{}, status.Errorf(codes.NotFound, "volume id %s does not exist in the volumes list", volID) +} + +func (s *VolumeStates) GetVolumeByName(volName string) (Volume, error) { + for _, volume := range s.Volumes { + if volume.VolName == volName { + return volume, nil + } + } + return Volume{}, status.Errorf(codes.NotFound, "volume name %s does not exist in the volumes list", volName) +} + +func (s *VolumeStates) AddVolume(v Volume) error { + for i, volume := range s.Volumes { + if volume.VolID == v.VolID { + s.Volumes[i] = v + return nil + } + } + s.Volumes = append(s.Volumes, v) + return s.dump() +} + +func (s *VolumeStates) DeleteVolume(volID string) error { + for i, volume := range s.Volumes { + if volume.VolID == volID { + s.Volumes = append(s.Volumes[:i], s.Volumes[i+1:]...) + return s.dump() + } + } + return nil +} diff --git a/k8s/cmd/commands/flags/csi_flags.go b/k8s/cmd/commands/flags/csi_flags.go new file mode 100644 index 00000000..777715cd --- /dev/null +++ b/k8s/cmd/commands/flags/csi_flags.go @@ -0,0 +1,38 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package flags + +import "github.com/spf13/cobra" + +var ( + // Endpoint is the endpoint of vineyard csi driver + Endpoint string + + // NodeID is the node id of vineyard csi driver + NodeID string + + // StateFilePath is the path of state file + StateFilePath string +) + +func ApplyCsiOpts(cmd *cobra.Command) { + cmd.Flags().StringVarP(&Endpoint, "endpoint", "f", "", + "the endpoint of vineyard csi driver") + cmd.Flags().StringVarP(&NodeID, "nodeid", "", "", + "the node id of vineyard csi driver") + cmd.Flags().StringVarP(&StateFilePath, "state-file-path", "", "/csi/state", + "the path of state file") +} diff --git a/k8s/cmd/commands/manager/manager.go b/k8s/cmd/commands/manager/manager.go index 94054d7d..b1ff343a 100644 --- a/k8s/cmd/commands/manager/manager.go +++ b/k8s/cmd/commands/manager/manager.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package start contains the start command of vineyard operator +// Package manager contains the start command of vineyard operator package manager import ( @@ -164,6 +164,14 @@ func startManager( log.Fatal(err, "unable to create recover controller") } + if err := (&controllers.CSIDriverReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + EventRecorder: mgr.GetEventRecorderFor("csidriver-controller"), + }).SetupWithManager(mgr); err != nil { + log.Fatal(err, "unable to create csidriver controller") + } + if flags.EnableWebhook { // register the webhooks of CRDs if err := (&v1alpha1.LocalObject{}).SetupWebhookWithManager(mgr); err != nil { @@ -187,6 +195,9 @@ func startManager( if err := (&v1alpha1.Recover{}).SetupWebhookWithManager(mgr); err != nil { log.Fatal(err, "unable to create recover webhook") } + if err := (&v1alpha1.CSIDriver{}).SetupWebhookWithManager(mgr); err != nil { + log.Fatal(err, "unable to create csidriver webhook") + } // register the assembly webhook log.Info("registering the assembly webhook") diff --git a/k8s/cmd/main.go b/k8s/cmd/main.go index 538003d9..e291d558 100644 --- a/k8s/cmd/main.go +++ b/k8s/cmd/main.go @@ -27,6 +27,7 @@ import ( gosdklog "github.com/v6d-io/v6d/go/vineyard/pkg/common/log" "github.com/v6d-io/v6d/k8s/cmd/commands/client" "github.com/v6d-io/v6d/k8s/cmd/commands/create" + "github.com/v6d-io/v6d/k8s/cmd/commands/csi" "github.com/v6d-io/v6d/k8s/cmd/commands/delete" "github.com/v6d-io/v6d/k8s/cmd/commands/deploy" "github.com/v6d-io/v6d/k8s/cmd/commands/flags" @@ -74,6 +75,7 @@ func init() { cmd.AddCommand(client.NewLsCmd()) cmd.AddCommand(client.NewGetCmd()) cmd.AddCommand(client.NewPutCmd()) + cmd.AddCommand(csi.NewCsiCmd()) } func main() { diff --git a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml new file mode 100644 index 00000000..01c18a1b --- /dev/null +++ b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml @@ -0,0 +1,94 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.0 + creationTimestamp: null + name: csidrivers.k8s.v6d.io +spec: + group: k8s.v6d.io + names: + kind: CSIDriver + listKind: CSIDriverList + plural: csidrivers + singular: csidriver + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + clusters: + items: + properties: + name: + default: "" + type: string + namespace: + default: "" + type: string + type: object + type: array + enableToleration: + default: false + type: boolean + image: + default: vineyardcloudnative/vineyard-csi-driver + type: string + imagePullPolicy: + default: IfNotPresent + type: string + sidecar: + default: + attacherImage: registry.k8s.io/sig-storage/csi-attacher:v4.0.0 + enableTopology: false + imagePullPolicy: Always + livenessProbeImage: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 + nodeRegistrarImage: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0 + provisionerImage: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + properties: + attacherImage: + default: registry.k8s.io/sig-storage/csi-attacher:v4.0.0 + type: string + enableTopology: + default: false + type: boolean + imagePullPolicy: + default: Always + type: string + livenessProbeImage: + default: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 + type: string + nodeRegistrarImage: + default: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0 + type: string + provisionerImage: + default: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + type: string + type: object + storageClassName: + default: vineyard-csi + type: string + volumeBindingMode: + default: WaitForFirstConsumer + type: string + type: object + status: + properties: + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/k8s/config/crd/kustomization.yaml b/k8s/config/crd/kustomization.yaml index 282bd62f..54a1d04d 100644 --- a/k8s/config/crd/kustomization.yaml +++ b/k8s/config/crd/kustomization.yaml @@ -9,6 +9,7 @@ resources: - bases/k8s.v6d.io_sidecars.yaml - bases/k8s.v6d.io_backups.yaml - bases/k8s.v6d.io_recovers.yaml +- bases/k8s.v6d.io_csidrivers.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -20,7 +21,8 @@ patchesStrategicMerge: - patches/webhook_in_operations.yaml - patches/webhook_in_sidecars.yaml - patches/webhook_in_backups.yaml -#- patches/webhook_in_recovers.yaml +- patches/webhook_in_recovers.yaml +- patches/webhook_in_csidrivers.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. @@ -31,7 +33,9 @@ patchesStrategicMerge: - patches/cainjection_in_operations.yaml - patches/cainjection_in_sidecars.yaml - patches/cainjection_in_backups.yaml -#- patches/cainjection_in_recovers.yaml +- patches/cainjection_in_recovers.yaml +- patches/cainjection_in_csidrivers.yaml +#- patches/cainjection_in_csidrivers.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/k8s/config/crd/patches/cainjection_in_csidrivers.yaml b/k8s/config/crd/patches/cainjection_in_csidrivers.yaml new file mode 100644 index 00000000..6e5108bb --- /dev/null +++ b/k8s/config/crd/patches/cainjection_in_csidrivers.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: csidrivers.k8s.v6d.io diff --git a/k8s/config/crd/patches/cainjection_in_k8s_recovers.yaml b/k8s/config/crd/patches/cainjection_in_recovers.yaml similarity index 100% rename from k8s/config/crd/patches/cainjection_in_k8s_recovers.yaml rename to k8s/config/crd/patches/cainjection_in_recovers.yaml diff --git a/k8s/config/crd/patches/webhook_in_csidrivers.yaml b/k8s/config/crd/patches/webhook_in_csidrivers.yaml new file mode 100644 index 00000000..a30d15ea --- /dev/null +++ b/k8s/config/crd/patches/webhook_in_csidrivers.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: csidrivers.k8s.v6d.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/k8s/config/crd/patches/webhook_in_k8s_recovers.yaml b/k8s/config/crd/patches/webhook_in_recovers.yaml similarity index 100% rename from k8s/config/crd/patches/webhook_in_k8s_recovers.yaml rename to k8s/config/crd/patches/webhook_in_recovers.yaml diff --git a/k8s/config/manager/manager.yaml b/k8s/config/manager/manager.yaml index 519cf333..49e486b7 100644 --- a/k8s/config/manager/manager.yaml +++ b/k8s/config/manager/manager.yaml @@ -11,6 +11,7 @@ metadata: name: controller-manager namespace: system labels: + k8s.v6d.io/instance: vineyard-operator control-plane: controller-manager spec: selector: diff --git a/k8s/config/rbac/k8s_csidriver_editor_role.yaml b/k8s/config/rbac/k8s_csidriver_editor_role.yaml new file mode 100644 index 00000000..a98ac668 --- /dev/null +++ b/k8s/config/rbac/k8s_csidriver_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit csidrivers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csidriver-editor-role +rules: +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers/status + verbs: + - get diff --git a/k8s/config/rbac/k8s_csidriver_viewer_role.yaml b/k8s/config/rbac/k8s_csidriver_viewer_role.yaml new file mode 100644 index 00000000..c8aa1c70 --- /dev/null +++ b/k8s/config/rbac/k8s_csidriver_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view csidrivers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csidriver-viewer-role +rules: +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers + verbs: + - get + - list + - watch +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers/status + verbs: + - get diff --git a/k8s/config/rbac/manager_account.yaml b/k8s/config/rbac/manager_account.yaml index 9f54994d..e1f683cf 100644 --- a/k8s/config/rbac/manager_account.yaml +++ b/k8s/config/rbac/manager_account.yaml @@ -1,10 +1,8 @@ - --- apiVersion: v1 kind: ServiceAccount metadata: name: manager - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -17,7 +15,6 @@ roleRef: subjects: - kind: ServiceAccount name: manager - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -30,7 +27,6 @@ roleRef: subjects: - kind: ServiceAccount name: manager - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -43,8 +39,6 @@ roleRef: subjects: - kind: ServiceAccount name: manager - - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/k8s/config/rbac/role.yaml b/k8s/config/rbac/role.yaml index 90efe900..97eba3d4 100644 --- a/k8s/config/rbac/role.yaml +++ b/k8s/config/rbac/role.yaml @@ -23,6 +23,14 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -34,6 +42,13 @@ rules: - list - patch - update + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/finalizers + verbs: + - patch - apiGroups: - "" resources: @@ -45,6 +60,13 @@ rules: - list - patch - update + - watch +- apiGroups: + - "" + resources: + - persistentvolumes/finalizers + verbs: + - patch - apiGroups: - "" resources: @@ -90,6 +112,16 @@ rules: - delete - get - update +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -127,6 +159,14 @@ rules: - get - list - update +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch - apiGroups: - k8s.v6d.io resources: @@ -147,6 +187,32 @@ rules: - get - patch - update +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers/finalizers + verbs: + - update +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers/status + verbs: + - get + - patch + - update - apiGroups: - k8s.v6d.io resources: @@ -285,3 +351,71 @@ rules: - get - patch - update +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - get + - list + - patch + - update + - watch diff --git a/k8s/config/samples/k8s_v1alpha1_csidriver.yaml b/k8s/config/samples/k8s_v1alpha1_csidriver.yaml new file mode 100644 index 00000000..e344eb85 --- /dev/null +++ b/k8s/config/samples/k8s_v1alpha1_csidriver.yaml @@ -0,0 +1,10 @@ +apiVersion: k8s.v6d.io/v1alpha1 +kind: CSIDriver +metadata: + name: csidriver-sample +spec: + clusters: + - namespace: vineyard-system + name: vineyardd-sample + - namespace: default + name: vineyardd-sample \ No newline at end of file diff --git a/k8s/config/webhook/manifests.yaml b/k8s/config/webhook/manifests.yaml index ff88f70e..f815a52b 100644 --- a/k8s/config/webhook/manifests.yaml +++ b/k8s/config/webhook/manifests.yaml @@ -231,6 +231,26 @@ webhooks: resources: - backups sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-k8s-v6d-io-v1alpha1-csidriver + failurePolicy: Fail + name: vcsidriver.kb.io + rules: + - apiGroups: + - k8s.v6d.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - csidrivers + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/k8s/controllers/k8s/csidriver_controller.go b/k8s/controllers/k8s/csidriver_controller.go new file mode 100644 index 00000000..ef6d57cb --- /dev/null +++ b/k8s/controllers/k8s/csidriver_controller.go @@ -0,0 +1,219 @@ +/** Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apache/skywalking-swck/operator/pkg/kubernetes" + "github.com/pkg/errors" + + k8sv1alpha1 "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1" + v1alpha1 "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1" + "github.com/v6d-io/v6d/k8s/pkg/log" + "github.com/v6d-io/v6d/k8s/pkg/templates" +) + +// CSIDriverReconciler reconciles a CSIDriver object +type CSIDriverReconciler struct { + client.Client + record.EventRecorder + Scheme *runtime.Scheme +} + +type StorageConfig struct { + Namespace string + Name string + VolumeBindingMode string +} + +//+kubebuilder:rbac:groups=k8s.v6d.io,resources=csidrivers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=k8s.v6d.io,resources=csidrivers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=k8s.v6d.io,resources=csidrivers/finalizers,verbs=update +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update +//+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update +//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +//+kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;watch;update;patch;create;delete +//+kubebuilder:rbac:groups="",resources=persistentvolumes/finalizers,verbs=patch +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims/finalizers,verbs=patch +//+kubebuilder:rbac:groups=storage.k8s.io, resources=storageclasses, verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=storage.k8s.io, resources=csinodes, verbs=get;list;watch +//+kubebuilder:rbac:groups=storage.k8s.io,resources=volumeattachments,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=storage.k8s.io,resources=volumeattachments/status,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=csi.storage.k8s.io,resources=csinodeinfos,verbs=get;list;watch +//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotclasses,verbs=get;list;watch +//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch;create;update;delete +//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshots,verbs=get;list;watch;update + +func (r *CSIDriverReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithName("controllers").WithName("CSIDriver") + + csiDriver := &k8sv1alpha1.CSIDriver{} + if err := r.Client.Get(ctx, req.NamespacedName, csiDriver); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + logger.Info("Reconciling CSIDriver", "csiDriver", csiDriver) + + // check there is no more than one csi driver in the cluster + csidrivers := &k8sv1alpha1.CSIDriverList{} + if err := r.Client.List(ctx, csidrivers); err != nil { + return ctrl.Result{}, err + } + + if len(csidrivers.Items) > 1 { + logger.Error(nil, "There is already a csi driver in the cluster") + return ctrl.Result{}, nil + } + + // check the vineyard clusters are ready + if len(csiDriver.Spec.Clusters) == 0 { + logger.Error(nil, "No vineyard cluster is specified") + return ctrl.Result{}, nil + } + deployment := &appsv1.Deployment{} + for _, c := range csiDriver.Spec.Clusters { + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.Name}, deployment); err != nil { + return ctrl.Result{}, err + } + if deployment.Status.ReadyReplicas != *deployment.Spec.Replicas { + logger.Error(nil, fmt.Sprintf("Vineyard deployment %s/%s is not ready", c.Namespace, c.Name)) + return ctrl.Result{}, nil + } + } + + // get the namespace of the vineyard operator + deploymentLists := &appsv1.DeploymentList{} + if err := r.Client.List(ctx, deploymentLists, client.MatchingLabels{"k8s.v6d.io/instance": "vineyard-operator"}); err != nil { + return ctrl.Result{}, err + } + if len(deploymentLists.Items) != 1 { + log.Errorf(nil, "Only one vineyard operator is allowed in the specific namespace, but got %v", len(deploymentLists.Items)) + return ctrl.Result{}, nil + } + if deploymentLists.Items[0].Status.ReadyReplicas != *deploymentLists.Items[0].Spec.Replicas { + log.Errorf(nil, "Vineyard operator is not ready") + return ctrl.Result{}, nil + } + + // the csi driver should be in the same namespace as the vineyard operator + // for sharing the same service account + csiDriver.Namespace = deploymentLists.Items[0].Namespace + // create a csi driver + csiDriverApp := kubernetes.Application{ + Client: r.Client, + CR: csiDriver, + FileRepo: templates.Repo, + GVK: k8sv1alpha1.GroupVersion.WithKind("CSIDriver"), + Recorder: r.EventRecorder, + TmplFunc: map[string]interface{}{}, + } + if _, err := csiDriverApp.Apply(ctx, "csidriver/daemonset.yaml", logger, true); err != nil { + logger.Error(err, "failed to apply csidriver daemonset manifest") + return ctrl.Result{}, err + } + if _, err := csiDriverApp.Apply(ctx, "csidriver/deployment.yaml", logger, true); err != nil { + logger.Error(err, "failed to apply csidriver deployment manifest") + return ctrl.Result{}, err + } + for i := 0; i < len(csiDriver.Spec.Clusters); i++ { + csiDriverApp.TmplFunc["getStorageConfig"] = func() StorageConfig { + return StorageConfig{ + Namespace: csiDriver.Spec.Clusters[i].Namespace, + Name: csiDriver.Spec.Clusters[i].Name, + VolumeBindingMode: csiDriver.Spec.VolumeBindingMode, + } + } + + if _, err := csiDriverApp.Apply(ctx, "csidriver/storageclass.yaml", logger, true); err != nil { + logger.Error(err, "failed to apply csidriver manifests") + return ctrl.Result{}, err + } + } + + if err := r.UpdateStatus(ctx, csiDriver); err != nil { + logger.Error(err, "Failed to update the status", "CSIDriver", csiDriver) + } + // reconcile every minute + duration, _ := time.ParseDuration("1m") + return ctrl.Result{RequeueAfter: duration}, nil +} + +func (r *CSIDriverReconciler) UpdateStatus(ctx context.Context, csiDriver *k8sv1alpha1.CSIDriver) error { + depOK := false + daeOK := false + depName := csiDriver.Name + "-csi-driver" + ns := csiDriver.Namespace + // check if the csi driver deployment is ready + dep := &appsv1.Deployment{} + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: ns, Name: depName}, dep); err != nil { + return errors.Wrap(err, "failed to get csi driver deployment") + } + if dep.Status.ReadyReplicas == *dep.Spec.Replicas { + depOK = true + } + + // check if the csi nodes daemonset is ready + daeName := csiDriver.Name + "-csi-nodes" + dae := &appsv1.DaemonSet{} + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: ns, Name: daeName}, dae); err != nil { + return errors.Wrap(err, "failed to get csi driver daemonset") + } + if dae.Status.NumberReady == dae.Status.DesiredNumberScheduled { + daeOK = true + } + + state := k8sv1alpha1.CSIDriverRunning + if !depOK || !daeOK { + state = k8sv1alpha1.CSIDriverPending + } + + status := &k8sv1alpha1.CSIDriverStatus{ + State: state, + } + if err := ApplyStatueUpdate(ctx, r.Client, csiDriver, r.Status(), + func(c *k8sv1alpha1.CSIDriver) (error, *k8sv1alpha1.CSIDriver) { + csiDriver.Status = *status + csiDriver.Kind = "CSIDriver" + + if err := kubernetes.ApplyOverlay(csiDriver, &v1alpha1.CSIDriver{Status: *status}); err != nil { + return errors.Wrap(err, "failed to overlay csidriver's status"), nil + } + return nil, csiDriver + }, + ); err != nil { + return errors.Wrap(err, "failed to update status") + } + + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CSIDriverReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&k8sv1alpha1.CSIDriver{}). + Complete(r) +} diff --git a/k8s/csitest.Dockerfile b/k8s/csitest.Dockerfile new file mode 100644 index 00000000..3b343423 --- /dev/null +++ b/k8s/csitest.Dockerfile @@ -0,0 +1,25 @@ +# Build the manager binary +FROM golang:1.19-buster as builder + +ENV GO111MODULE=on +ENV CGO_ENABLED=0 +ENV GOOS=linux + +WORKDIR /workspace + +# Copy the vineyard Go SDK +COPY go/ go/ + +# Copy the Go source for vineyardctl +COPY k8s/ k8s/ + +WORKDIR /workspace/k8s + +RUN go mod download + +# Update the working directory +WORKDIR /workspace/k8s/cmd/commands/csi + +USER root + +CMD ["go", "test", "-run", "./..."] \ No newline at end of file diff --git a/k8s/examples/vineyard-csidriver/Dockerfile b/k8s/examples/vineyard-csidriver/Dockerfile new file mode 100644 index 00000000..728442cc --- /dev/null +++ b/k8s/examples/vineyard-csidriver/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.10 + +RUN pip3 install --no-cache-dir pandas requests scikit-learn numpy vineyard + +WORKDIR / + +ARG APP +ENV APP ${APP} + +COPY ${APP} /${APP} + + diff --git a/k8s/examples/vineyard-csidriver/Makefile b/k8s/examples/vineyard-csidriver/Makefile new file mode 100644 index 00000000..b82b6ecc --- /dev/null +++ b/k8s/examples/vineyard-csidriver/Makefile @@ -0,0 +1,22 @@ +docker-build: + docker build prepare-data/ -f Dockerfile \ + --build-arg APP=prepare-data.py \ + -t prepare-data + + docker build preprocess/ -f Dockerfile \ + --build-arg APP=preprocess.py \ + -t preprocess-data + + docker build train/ -f Dockerfile \ + --build-arg APP=train.py \ + -t train-data + + docker build test/ -f Dockerfile \ + --build-arg APP=test.py \ + -t test-data + +load-images: + kind load docker-image prepare-data + kind load docker-image preprocess-data + kind load docker-image train-data + kind load docker-image test-data \ No newline at end of file diff --git a/k8s/examples/vineyard-csidriver/README.md b/k8s/examples/vineyard-csidriver/README.md new file mode 100644 index 00000000..4ff001d8 --- /dev/null +++ b/k8s/examples/vineyard-csidriver/README.md @@ -0,0 +1,205 @@ +# Kubeflow Example with Vineyard CSI Driver + +## Prerequisites + +If you don't have a kubernetes cluster by hand, you can use [kind](https://kind.sigs.k8s.io/) to create a kubernetes cluster: + +```bash +$ cat < /proc/sys/vm/drop_caches') + data_multiplier = os.environ.get('DATA_MULTIPLIER', 1) + st = time.time() + df = pd.read_pickle('/data/df_{0}.pkl'.format(data_multiplier)) + + ed = time.time() + print('##################################') + print('read dataframe pickle time: ', ed - st) + + df = df.drop(df[(df['GrLivArea']>4800)].index) + + """ The following part will need large memory usage, disable for benchmark + del df + + # Define the categorical feature columns + categorical_features = df_preocessed.select_dtypes(include='object').columns + + # Create the column transformer for one-hot encoding + preprocessor = ColumnTransformer( + transformers=[('encoder', OneHotEncoder(sparse=False, handle_unknown='ignore'), categorical_features)], + remainder='passthrough' + ) + + # Preprocess the features using the column transformer + one_hot_df = preprocessor.fit_transform(df_preocessed) + + # Get the column names for the encoded features + encoded_feature_names = preprocessor.named_transformers_['encoder'].get_feature_names_out(categorical_features) + + columns = list(encoded_feature_names) + list(df_preocessed.select_dtypes(exclude='object').columns) + + del df_preocessed + + # Concatenate the encoded features with the original numerical features + df = pd.DataFrame(one_hot_df, columns=columns) + + del one_hot_df + """ + + X = df.drop('SalePrice', axis=1) # Features + y = df['SalePrice'] # Target variable + + del df + + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1) + + del X, y + + st = time.time() + enable_vineyard = os.environ.get('ENABLE_VINEYARD', False) + if enable_vineyard: + vineyard.csi.write(X_train, "/data/x_train.pkl") + vineyard.csi.write(X_test, "/data/x_test.pkl") + vineyard.csi.write(y_train, "/data/y_train.pkl") + vineyard.csi.write(y_test, "/data/y_test.pkl") + else: + X_train.to_pickle('/data/x_train.pkl') + X_test.to_pickle('/data/x_test.pkl') + y_train.to_pickle('/data/y_train.pkl') + y_test.to_pickle('/data/y_test.pkl') + + ed = time.time() + print('##################################') + print('write training and testing data time: ', ed - st) + + +if __name__ == '__main__': + st = time.time() + print('Preprocessing data...') + preprocess_data() + ed = time.time() + print('##################################') + print('Preprocessing data time: ', ed - st) diff --git a/k8s/examples/vineyard-csidriver/rbac.yaml b/k8s/examples/vineyard-csidriver/rbac.yaml new file mode 100644 index 00000000..3d791138 --- /dev/null +++ b/k8s/examples/vineyard-csidriver/rbac.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pipeline-runner +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pipeline-runner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "create", "update", "list", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "patch", "create", "update", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pipeline-runner-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pipeline-runner-role +subjects: + - kind: ServiceAccount + name: pipeline-runner + namespace: default \ No newline at end of file diff --git a/k8s/examples/vineyard-csidriver/test/test.py b/k8s/examples/vineyard-csidriver/test/test.py new file mode 100644 index 00000000..868d4adc --- /dev/null +++ b/k8s/examples/vineyard-csidriver/test/test.py @@ -0,0 +1,42 @@ +import os +import time + +from sklearn.metrics import mean_squared_error + +import joblib +import pandas as pd +import vineyard + +def test_model(): + os.system('echo 3 > /proc/sys/vm/drop_caches') + enable_vineyard = os.environ.get('ENABLE_VINEYARD', False) + st = time.time() + if enable_vineyard: + x_test_data = vineyard.csi.read("/data/x_test.pkl") + y_test_data = vineyard.csi.read("/data/y_test.pkl") + else: + x_test_data = pd.read_pickle("/data/x_test.pkl") + y_test_data = pd.read_pickle("/data/y_test.pkl") + #delete the x_test.pkl and y_test.pkl + os.remove("/data/x_test.pkl") + os.remove("/data/y_test.pkl") + ed = time.time() + print('##################################') + print('read x_test and y_test execution time: ', ed - st) + + model = joblib.load("/data/model.pkl") + y_pred = model.predict(x_test_data) + + err = mean_squared_error(y_test_data, y_pred) + + with open('/data/output.txt', 'a') as f: + f.write(str(err)) + + +if __name__ == '__main__': + st = time.time() + print('Testing model...') + test_model() + ed = time.time() + print('##################################') + print('Testing model data time: ', ed - st) diff --git a/k8s/examples/vineyard-csidriver/train/train.py b/k8s/examples/vineyard-csidriver/train/train.py new file mode 100644 index 00000000..2cca3b40 --- /dev/null +++ b/k8s/examples/vineyard-csidriver/train/train.py @@ -0,0 +1,40 @@ +import os +import time + +from sklearn.linear_model import LinearRegression + +import joblib +import pandas as pd +import vineyard + + +def train_model(): + os.system('echo 3 > /proc/sys/vm/drop_caches') + st = time.time() + enable_vineyard = os.environ.get('ENABLE_VINEYARD', False) + if enable_vineyard: + x_train_data = vineyard.csi.read("/data/x_train.pkl") + y_train_data = vineyard.csi.read("/data/y_train.pkl") + else: + x_train_data = pd.read_pickle("/data/x_train.pkl") + y_train_data = pd.read_pickle("/data/y_train.pkl") + # delete the x_train.pkl and y_train.pkl + os.remove("/data/x_train.pkl") + os.remove("/data/y_train.pkl") + ed = time.time() + print('##################################') + print('read x_train and y_train data time: ', ed - st) + + model = LinearRegression() + model.fit(x_train_data, y_train_data) + + joblib.dump(model, '/data/model.pkl') + + +if __name__ == '__main__': + st = time.time() + print('Training model...') + train_model() + ed = time.time() + print('##################################') + print('Training model data time: ', ed - st) diff --git a/k8s/go.mod b/k8s/go.mod index 01935d1d..6c3db166 100644 --- a/k8s/go.mod +++ b/k8s/go.mod @@ -7,8 +7,12 @@ require ( github.com/argoproj/argo-workflows/v3 v3.3.10 github.com/avast/retry-go v3.0.0+incompatible github.com/cert-manager/cert-manager v1.8.0 + github.com/container-storage-interface/spec v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 + github.com/google/uuid v1.3.0 + github.com/kubernetes-csi/csi-lib-utils v0.0.0-00010101000000-000000000000 + github.com/kubernetes-csi/csi-test/v4 v4.4.0 github.com/olekukonko/tablewriter v0.0.4 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.4.0 @@ -17,6 +21,7 @@ require ( github.com/v6d-io/v6d/go/vineyard v0.0.0-00010101000000-000000000000 go.uber.org/multierr v1.9.0 go.uber.org/zap v1.24.0 + google.golang.org/grpc v1.53.0 k8s.io/api v0.25.10 k8s.io/apiextensions-apiserver v0.25.10 k8s.io/apimachinery v0.25.10 @@ -24,6 +29,7 @@ require ( k8s.io/component-base v0.25.10 k8s.io/component-helpers v0.25.10 k8s.io/kubernetes v1.25.10 + k8s.io/mount-utils v0.25.10 sigs.k8s.io/controller-runtime v0.13.0 sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 sigs.k8s.io/kustomize/kyaml v0.13.9 @@ -63,7 +69,6 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect @@ -122,7 +127,6 @@ require ( gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect @@ -134,7 +138,6 @@ require ( k8s.io/klog/v2 v2.70.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect k8s.io/kube-scheduler v0.25.10 // indirect - k8s.io/mount-utils v0.25.10 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect sigs.k8s.io/gateway-api v0.4.1 // indirect @@ -149,8 +152,9 @@ replace ( //github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.9.5+incompatible //github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful v2.9.5+incompatible github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.4.2 - github.com/v6d-io/v6d/go/vineyard => ../go/vineyard // these are needed since k8s.io/kubernetes cites v0.0.0 for these in its go.mod + github.com/kubernetes-csi/csi-lib-utils => github.com/kubernetes-csi/csi-lib-utils v0.11.0 + github.com/v6d-io/v6d/go/vineyard => ../go/vineyard k8s.io/api => k8s.io/api v0.25.10 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.25.10 k8s.io/apimachinery => k8s.io/apimachinery v0.25.10 diff --git a/k8s/go.sum b/k8s/go.sum index 2552f00c..d1a57186 100644 --- a/k8s/go.sum +++ b/k8s/go.sum @@ -135,6 +135,9 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= +github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -411,6 +414,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/csi-lib-utils v0.11.0 h1:FHWOBtAZBA/hVk7v/qaXgG9Sxv0/n06DebPFuDwumqg= +github.com/kubernetes-csi/csi-lib-utils v0.11.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0= +github.com/kubernetes-csi/csi-test/v4 v4.4.0 h1:r0mnAwDURI24Vw3a/LyA/ga11yD5ZGuU7+REO35Na9s= +github.com/kubernetes-csi/csi-test/v4 v4.4.0/go.mod h1:t1RzseMZJKy313nezI/d7TolbbiKpUZM3SXQvXxOX0w= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -782,6 +789,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1075,6 +1083,7 @@ google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1181,8 +1190,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1215,7 +1224,9 @@ k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= @@ -1254,4 +1265,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= \ No newline at end of file +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/k8s/pkg/schedulers/scheduling_strategy.go b/k8s/pkg/schedulers/scheduling_strategy.go index 140a11ea..b8c8c0b7 100644 --- a/k8s/pkg/schedulers/scheduling_strategy.go +++ b/k8s/pkg/schedulers/scheduling_strategy.go @@ -131,6 +131,7 @@ func (b *BestEffortStrategy) TrackingChunksByCRD() *BestEffortStrategy { } locations := b.GetLocationsByLocalObject(localObjects) nchunks, nodes := b.GetObjectInfo(locations, len(localObjects), b.replica) + b.locations = locations b.nchunks = nchunks b.nodes = nodes diff --git a/k8s/pkg/templates/csidriver/daemonset.yaml b/k8s/pkg/templates/csidriver/daemonset.yaml new file mode 100644 index 00000000..db4375a8 --- /dev/null +++ b/k8s/pkg/templates/csidriver/daemonset.yaml @@ -0,0 +1,110 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ .Name }}-csi-nodes + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: {{ .Name }}-csi-nodes + template: + metadata: + labels: + app: {{ .Name }}-csi-nodes + app.vineyard.io/name: {{ .Name }} + app.vineyard.io/role: csi-nodes + spec: + containers: + - command: + - /vineyardctl + args: + - csi + - --endpoint=$(CSI_ENDPOINT) + - --nodeid=$(KUBE_NODE_NAME) + - --state-file-path=/csi/state + - --verbose + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: {{ .Spec.Image }} + imagePullPolicy: {{ .Spec.ImagePullPolicy }} + name: vineyard-csi-driver + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - mountPath: /var/run/vineyard-kubernetes + name: vineyard-sockets + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.vineyard.v6d.io/csi.sock + image: {{ .Spec.Sidecar.NodeRegistrarImage }} + imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }} + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/csi.vineyard.v6d.io-reg.sock /csi/csi.sock + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - mountPath: /var/run/vineyard-kubernetes + name: vineyard-sockets + - args: + - --csi-address=/csi/csi.sock + - --v=5 + image: {{ .Spec.Sidecar.LivenessProbeImage }} + imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }} + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/run/vineyard-kubernetes + name: vineyard-sockets + {{- if .Spec.EnableToleration }} + tolerations: + - operator: Exists + {{- end }} + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.vineyard.v6d.io/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /var/run/vineyard-kubernetes + type: Directory + name: vineyard-sockets diff --git a/k8s/pkg/templates/csidriver/deployment.yaml b/k8s/pkg/templates/csidriver/deployment.yaml new file mode 100644 index 00000000..c0a7051f --- /dev/null +++ b/k8s/pkg/templates/csidriver/deployment.yaml @@ -0,0 +1,104 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Name }}-csi-driver + namespace: {{ .Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Name }}-csi-driver + template: + metadata: + labels: + app: {{ .Name }}-csi-driver + app.vineyard.io/name: {{ .Name }} + app.vineyard.io/role: csi-driver + spec: + containers: + - name: vineyard-csi-driver + image: {{ .Spec.Image }} + imagePullPolicy: {{ .Spec.ImagePullPolicy }} + command: + - /vineyardctl + args: + - csi + - --endpoint=$(CSI_ENDPOINT) + - --nodeid=$(KUBE_NODE_NAME) + - --state-file-path=/csi/state + - --verbose + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + #Sidecar: livenessprobe + - name: liveness-probe + image: {{ .Spec.Sidecar.LivenessProbeImage }} + imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }} + args: + - "--csi-address=/csi/csi.sock" + - "--v=5" + volumeMounts: + - name: plugin-dir + mountPath: /csi + #Sidecar: csi-provisioner + - name: csi-provisioner + image: {{ .Spec.Sidecar.ProvisionerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + {{- if .Spec.Sidecar.EnableTopology }} + - "--feature-gates=Topology=True" + {{- end }} + - "--extra-create-metadata" + env: + - name: ADDRESS + value: unix:///csi/csi.sock + imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + #Sidecar: csi-attacher + - name: csi-attacher + image: {{ .Spec.Sidecar.AttacherImage }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.v6d.io/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/k8s/pkg/templates/csidriver/storageclass.yaml b/k8s/pkg/templates/csidriver/storageclass.yaml new file mode 100644 index 00000000..a5024d5f --- /dev/null +++ b/k8s/pkg/templates/csidriver/storageclass.yaml @@ -0,0 +1,10 @@ +{{- $storage := getStorageConfig }} +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ $storage.Namespace -}}.{{- $storage.Name -}}.csi +provisioner: csi.vineyard.v6d.io +parameters: + k8s.v6d.io/vineyard/namespace: {{ $storage.Namespace }} + k8s.v6d.io/vineyard/name: {{ $storage.Name }} +volumeBindingMode: {{ $storage.VolumeBindingMode }} diff --git a/k8s/pkg/templates/template.go b/k8s/pkg/templates/template.go index 187c991c..032d646e 100644 --- a/k8s/pkg/templates/template.go +++ b/k8s/pkg/templates/template.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" ) -//go:embed vineyardd etcd operation sidecar backup recover certmanager +//go:embed vineyardd etcd operation sidecar backup recover certmanager csidriver var fs embed.FS // ReadFile reads a file from the embed.FS diff --git a/k8s/test/e2e/Makefile b/k8s/test/e2e/Makefile index b36e82f7..339b01e0 100644 --- a/k8s/test/e2e/Makefile +++ b/k8s/test/e2e/Makefile @@ -407,7 +407,8 @@ unittest: prepare-e2e-test build-local-cluster load-vineyardd-image @make -C ${E2E_DIR} install-vineyard-operator @echo "Running vineyardctl unit test.." # run all unit test except deploy and scheduler in parallel - @export KUBECONFIG=/tmp/e2e-k8s.config && cd ${CMD_DIR}/commands && go test `go list ./... | grep -Ev "deploy|schedule"` +# dsiable vineyard csi driver test here + @export KUBECONFIG=/tmp/e2e-k8s.config && cd ${CMD_DIR}/commands && go test `go list ./... | grep -Ev "deploy|schedule|csi"` # deploy test @export KUBECONFIG="/tmp/e2e-k8s.config" && cd ${CMD_DIR}/commands/deploy && \ go test -run first && go test -run second && go test -run third From 226f839d908e1168dc9f81f4ba967c487f483156 Mon Sep 17 00:00:00 2001 From: Ye Cao Date: Mon, 18 Sep 2023 10:34:19 +0800 Subject: [PATCH 2/4] * Move the csi/pkg to top-level pkg. * Update the golang-1.19 to gcr.io/distroless/base:debug as csidriver's base image. * Improve the csidriver example readme. Signed-off-by: Ye Cao --- k8s/Makefile | 4 +-- k8s/apis/k8s/v1alpha1/csidriver_types.go | 5 +++ k8s/cmd/commands/csi/csi.go | 4 +-- k8s/cmd/commands/csi/csi_test.go | 4 +-- .../crd/bases/k8s.v6d.io_csidrivers.yaml | 3 ++ k8s/config/crd/kustomization.yaml | 1 - k8s/examples/vineyard-csidriver/README.md | 32 +++++++++---------- .../pipeline-with-vineyard.yaml | 6 ++-- .../preprocess/preprocess.py | 6 ++-- k8s/examples/vineyard-csidriver/test/test.py | 6 ++-- .../vineyard-csidriver/train/train.py | 4 +-- .../csi/pkg => pkg/csidriver}/controller.go | 2 +- .../csi/pkg => pkg/csidriver}/driver.go | 7 ++-- .../csi/pkg => pkg/csidriver}/identity.go | 2 +- .../csi/pkg => pkg/csidriver}/node.go | 2 +- .../csi/pkg => pkg/csidriver}/state.go | 3 +- k8s/pkg/templates/csidriver/daemonset.yaml | 2 ++ k8s/pkg/templates/csidriver/deployment.yaml | 2 ++ 18 files changed, 53 insertions(+), 42 deletions(-) rename k8s/{cmd/commands/csi/pkg => pkg/csidriver}/controller.go (99%) rename k8s/{cmd/commands/csi/pkg => pkg/csidriver}/driver.go (95%) rename k8s/{cmd/commands/csi/pkg => pkg/csidriver}/identity.go (99%) rename k8s/{cmd/commands/csi/pkg => pkg/csidriver}/node.go (99%) rename k8s/{cmd/commands/csi/pkg => pkg/csidriver}/state.go (99%) diff --git a/k8s/Makefile b/k8s/Makefile index be8a06a2..17a44824 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -237,11 +237,11 @@ csidriver: cd .. && \ if docker build --help | grep -q load; then \ docker build --load -f k8s/Dockerfile . \ - --build-arg BASE_IMAGE=golang:1.19-buster \ + --build-arg BASE_IMAGE=gcr.io/distroless/base:debug \ -t $(VINEYARD_CSI_IMAGE); \ else \ docker build -f k8s/Dockerfile . \ - --build-arg BASE_IMAGE=golang:1.19-buster \ + --build-arg BASE_IMAGE=gcr.io/distroless/base:debug \ -t $(VINEYARD_CSI_IMAGE); \ fi diff --git a/k8s/apis/k8s/v1alpha1/csidriver_types.go b/k8s/apis/k8s/v1alpha1/csidriver_types.go index b3027ef5..3c57679b 100644 --- a/k8s/apis/k8s/v1alpha1/csidriver_types.go +++ b/k8s/apis/k8s/v1alpha1/csidriver_types.go @@ -102,6 +102,11 @@ type CSIDriverSpec struct { // +kubebuilder:validation:Required // +kubebuilder:default:=false EnableToleration bool `json:"enableToleration,omitempty"` + + // EnableDebugLog is the flag to enable debug log for the csi driver + // +kubebuilder:validation:Required + // +kubebuilder:default:=false + EnableDebugLog bool `json:"enableDebugLog,omitempty"` } // CSIDriverStatus defines the observed state of CSIDriver diff --git a/k8s/cmd/commands/csi/csi.go b/k8s/cmd/commands/csi/csi.go index 0ff38be1..b0afc46c 100644 --- a/k8s/cmd/commands/csi/csi.go +++ b/k8s/cmd/commands/csi/csi.go @@ -20,9 +20,9 @@ package csi import ( "github.com/spf13/cobra" - "github.com/v6d-io/v6d/k8s/cmd/commands/csi/pkg" "github.com/v6d-io/v6d/k8s/cmd/commands/flags" "github.com/v6d-io/v6d/k8s/cmd/commands/util" + "github.com/v6d-io/v6d/k8s/pkg/csidriver" ) var csiExample = util.Examples(` @@ -36,7 +36,7 @@ var csiCmd = &cobra.Command{ Example: csiExample, Run: func(cmd *cobra.Command, args []string) { util.AssertNoArgs(cmd, args) - d := pkg.NewDriver(flags.NodeID, flags.Endpoint) + d := csidriver.NewDriver(flags.NodeID, flags.Endpoint) d.Run() }, } diff --git a/k8s/cmd/commands/csi/csi_test.go b/k8s/cmd/commands/csi/csi_test.go index f0925f17..68cfb30b 100644 --- a/k8s/cmd/commands/csi/csi_test.go +++ b/k8s/cmd/commands/csi/csi_test.go @@ -25,8 +25,8 @@ import ( "testing" "github.com/kubernetes-csi/csi-test/v4/pkg/sanity" - "github.com/v6d-io/v6d/k8s/cmd/commands/csi/pkg" "github.com/v6d-io/v6d/k8s/cmd/commands/flags" + "github.com/v6d-io/v6d/k8s/pkg/csidriver" ) func TestVineyardCSIDriver(t *testing.T) { @@ -41,7 +41,7 @@ func TestVineyardCSIDriver(t *testing.T) { flags.Endpoint = csiEndpoint flags.NodeID = "test-node-id" - vineyardSocket := filepath.Join(pkg.VineyardSocketPrefix, pkg.VineyardSocket) + vineyardSocket := filepath.Join(csidriver.VineyardSocketPrefix, csidriver.VineyardSocket) if _, err := os.OpenFile(vineyardSocket, os.O_CREATE|os.O_RDONLY, 0666); err != nil { t.Errorf("failed to open vineyard socket file %s: %v", vineyardSocket, err) } diff --git a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml index 01c18a1b..e1b8e243 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml @@ -38,6 +38,9 @@ spec: type: string type: object type: array + enableDebugLog: + default: false + type: boolean enableToleration: default: false type: boolean diff --git a/k8s/config/crd/kustomization.yaml b/k8s/config/crd/kustomization.yaml index 54a1d04d..dda9bc5f 100644 --- a/k8s/config/crd/kustomization.yaml +++ b/k8s/config/crd/kustomization.yaml @@ -35,7 +35,6 @@ patchesStrategicMerge: - patches/cainjection_in_backups.yaml - patches/cainjection_in_recovers.yaml - patches/cainjection_in_csidrivers.yaml -#- patches/cainjection_in_csidrivers.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/k8s/examples/vineyard-csidriver/README.md b/k8s/examples/vineyard-csidriver/README.md index 4ff001d8..61989c0a 100644 --- a/k8s/examples/vineyard-csidriver/README.md +++ b/k8s/examples/vineyard-csidriver/README.md @@ -160,11 +160,11 @@ The time of argo workflow execution of the pipeline is as follows: ### Argo workflow duration -| data_multiplier | without vineyard | with vineyard | +| data scale | without vineyard | with vineyard | | --------------- | ---------------- | ------------- | -| 3000(8.5G) | 186s | 169s | -| 4000(12G) | 250s | 203s | -| 5000(15G) | 332s | 286s | +| 8500 Mi | 186s | 169s | +| 12000 Mi | 250s | 203s | +| 15000 Mi | 332s | 286s | Actually, the cost time of argo workflow is affected by lots of factors, such as the network, the cpu and memory of the cluster, the data volume, etc. So the time of argo workflow execution of the pipeline is not stable. But we can still find that the time of argo workflow execution of the pipeline with vineyard is shorter than that without vineyard. @@ -173,32 +173,32 @@ Also, we record the whole execution time via logs. The result is as follows: ### Actual execution time -| data_multiplier | without vineyard | with vineyard | +| data scale | without vineyard | with vineyard | | --------------- | ---------------- | ------------- | -| 3000 | 139.3s | 92.3s | -| 4000 | 204.3s | 131.1s | -| 5000 | 289.3s | 209.7s | +| 8500 Mi | 139.3s | 92.3s | +| 12000 Mi | 204.3s | 131.1s | +| 15000 Mi | 289.3s | 209.7s | According to the above results, we can find that the time of actual execution of the pipeline with vineyard is shorter than that without vineyard. To be specific, we record the write/read time of the following steps: ### Write time -| data_multiplier | without vineyard | with vineyard | +| data scale | without vineyard | with vineyard | | --------------- | ---------------- | ------------- | -| 3000 | 21s | 5.4s | -| 4000 | 26s | 7s | -| 5000 | 32.2s | 9.4s | +| 8500 Mi | 21s | 5.4s | +| 12000 Mi | 26s | 7s | +| 15000 Mi | 32.2s | 9.4s | From the above results, we can find that the time of write of the pipeline with vineyard is nearly 4 times shorter than that without vineyard. The reason is that the data is stored in the vineyard cluster, so it's actually a memory copy operation, which is faster than the write operation of the nfs volume. ### Read time(Delete the time of init data loading) -| data_multiplier | without vineyard | with vineyard | +| data scale | without vineyard | with vineyard | | --------------- | ---------------- | ------------- | -| 3000 | 36.7s | 0.02s | -| 4000 | 45.7s | 0.02s | -| 5000 | 128.6s | 0.04s | +| 8500 Mi | 36.7s | 0.02s | +| 12000 Mi | 45.7s | 0.02s | +| 15000 Mi | 128.6s | 0.04s | Based on the above results, we can find that the read time of vineyard is nearly a constant, which is not affected by the data scale. The reason is that the data is stored in the shared memory of vineyard cluster, so it's actually a pointer copy operation. diff --git a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml index 4a9a8430..1b943fc6 100644 --- a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml +++ b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml @@ -47,7 +47,7 @@ spec: securityContext: privileged: true env: - - name: ENABLE_VINEYARD + - name: WITH_VINEYARD value: "true" - name: DATA_MULTIPLIER value: "{{workflow.parameters.data_multiplier}}" @@ -87,7 +87,7 @@ spec: securityContext: privileged: true env: - - name: ENABLE_VINEYARD + - name: WITH_VINEYARD value: "true" - name: DATA_MULTIPLIER value: "{{workflow.parameters.data_multiplier}}" @@ -119,7 +119,7 @@ spec: securityContext: privileged: true env: - - name: ENABLE_VINEYARD + - name: WITH_VINEYARD value: "true" - name: DATA_MULTIPLIER value: "{{workflow.parameters.data_multiplier}}" diff --git a/k8s/examples/vineyard-csidriver/preprocess/preprocess.py b/k8s/examples/vineyard-csidriver/preprocess/preprocess.py index a01a2133..64237f0f 100644 --- a/k8s/examples/vineyard-csidriver/preprocess/preprocess.py +++ b/k8s/examples/vineyard-csidriver/preprocess/preprocess.py @@ -10,7 +10,7 @@ def preprocess_data(): - os.system('echo 3 > /proc/sys/vm/drop_caches') + os.system('sync; echo 3 > /proc/sys/vm/drop_caches') data_multiplier = os.environ.get('DATA_MULTIPLIER', 1) st = time.time() df = pd.read_pickle('/data/df_{0}.pkl'.format(data_multiplier)) @@ -59,8 +59,8 @@ def preprocess_data(): del X, y st = time.time() - enable_vineyard = os.environ.get('ENABLE_VINEYARD', False) - if enable_vineyard: + with_vineyard = os.environ.get('WITH_VINEYARD', False) + if with_vineyard: vineyard.csi.write(X_train, "/data/x_train.pkl") vineyard.csi.write(X_test, "/data/x_test.pkl") vineyard.csi.write(y_train, "/data/y_train.pkl") diff --git a/k8s/examples/vineyard-csidriver/test/test.py b/k8s/examples/vineyard-csidriver/test/test.py index 868d4adc..97d5d0da 100644 --- a/k8s/examples/vineyard-csidriver/test/test.py +++ b/k8s/examples/vineyard-csidriver/test/test.py @@ -8,10 +8,10 @@ import vineyard def test_model(): - os.system('echo 3 > /proc/sys/vm/drop_caches') - enable_vineyard = os.environ.get('ENABLE_VINEYARD', False) + os.system('sync; echo 3 > /proc/sys/vm/drop_caches') + with_vineyard = os.environ.get('WITH_VINEYARD', False) st = time.time() - if enable_vineyard: + if with_vineyard: x_test_data = vineyard.csi.read("/data/x_test.pkl") y_test_data = vineyard.csi.read("/data/y_test.pkl") else: diff --git a/k8s/examples/vineyard-csidriver/train/train.py b/k8s/examples/vineyard-csidriver/train/train.py index 2cca3b40..93a71c77 100644 --- a/k8s/examples/vineyard-csidriver/train/train.py +++ b/k8s/examples/vineyard-csidriver/train/train.py @@ -11,8 +11,8 @@ def train_model(): os.system('echo 3 > /proc/sys/vm/drop_caches') st = time.time() - enable_vineyard = os.environ.get('ENABLE_VINEYARD', False) - if enable_vineyard: + with_vineyard = os.environ.get('WITH_VINEYARD', False) + if with_vineyard: x_train_data = vineyard.csi.read("/data/x_train.pkl") y_train_data = vineyard.csi.read("/data/y_train.pkl") else: diff --git a/k8s/cmd/commands/csi/pkg/controller.go b/k8s/pkg/csidriver/controller.go similarity index 99% rename from k8s/cmd/commands/csi/pkg/controller.go rename to k8s/pkg/csidriver/controller.go index 2fb94877..bf0b7441 100644 --- a/k8s/cmd/commands/csi/pkg/controller.go +++ b/k8s/pkg/csidriver/controller.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package pkg +package csidriver import ( "context" diff --git a/k8s/cmd/commands/csi/pkg/driver.go b/k8s/pkg/csidriver/driver.go similarity index 95% rename from k8s/cmd/commands/csi/pkg/driver.go rename to k8s/pkg/csidriver/driver.go index 1a2dfd8d..96da0696 100644 --- a/k8s/cmd/commands/csi/pkg/driver.go +++ b/k8s/pkg/csidriver/driver.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package pkg +package csidriver import ( "context" @@ -57,8 +57,9 @@ func (d *Driver) Run() { vineyardCSI := NewVineyardCSI(flags.StateFilePath, d.nodeID) identity := NewIdentityServer() - opts := []grpc.ServerOption{ - grpc.UnaryInterceptor(logGRPC), + opts := []grpc.ServerOption{} + if flags.Verbose { + opts = append(opts, grpc.UnaryInterceptor(logGRPC)) } srv := grpc.NewServer(opts...) diff --git a/k8s/cmd/commands/csi/pkg/identity.go b/k8s/pkg/csidriver/identity.go similarity index 99% rename from k8s/cmd/commands/csi/pkg/identity.go rename to k8s/pkg/csidriver/identity.go index dd565aca..34fa9ee1 100644 --- a/k8s/cmd/commands/csi/pkg/identity.go +++ b/k8s/pkg/csidriver/identity.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package pkg +package csidriver import ( "context" diff --git a/k8s/cmd/commands/csi/pkg/node.go b/k8s/pkg/csidriver/node.go similarity index 99% rename from k8s/cmd/commands/csi/pkg/node.go rename to k8s/pkg/csidriver/node.go index 0deaad05..11966b5d 100644 --- a/k8s/cmd/commands/csi/pkg/node.go +++ b/k8s/pkg/csidriver/node.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package pkg +package csidriver import ( "context" diff --git a/k8s/cmd/commands/csi/pkg/state.go b/k8s/pkg/csidriver/state.go similarity index 99% rename from k8s/cmd/commands/csi/pkg/state.go rename to k8s/pkg/csidriver/state.go index 23ec6476..8336956c 100644 --- a/k8s/cmd/commands/csi/pkg/state.go +++ b/k8s/pkg/csidriver/state.go @@ -13,8 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - -package pkg +package csidriver import ( "encoding/json" diff --git a/k8s/pkg/templates/csidriver/daemonset.yaml b/k8s/pkg/templates/csidriver/daemonset.yaml index db4375a8..8e739cea 100644 --- a/k8s/pkg/templates/csidriver/daemonset.yaml +++ b/k8s/pkg/templates/csidriver/daemonset.yaml @@ -22,7 +22,9 @@ spec: - --endpoint=$(CSI_ENDPOINT) - --nodeid=$(KUBE_NODE_NAME) - --state-file-path=/csi/state + {{ if .Spec.EnableDebugLog }} - --verbose + {{ end }} env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock diff --git a/k8s/pkg/templates/csidriver/deployment.yaml b/k8s/pkg/templates/csidriver/deployment.yaml index c0a7051f..55d946fa 100644 --- a/k8s/pkg/templates/csidriver/deployment.yaml +++ b/k8s/pkg/templates/csidriver/deployment.yaml @@ -26,7 +26,9 @@ spec: - --endpoint=$(CSI_ENDPOINT) - --nodeid=$(KUBE_NODE_NAME) - --state-file-path=/csi/state + {{ if .Spec.EnableDebugLog }} - --verbose + {{ end }} env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock From e6fd19cc142d58225c7f2c1eda123dfb07151b74 Mon Sep 17 00:00:00 2001 From: Ye Cao Date: Mon, 18 Sep 2023 16:32:53 +0800 Subject: [PATCH 3/4] * Move the csi driver test to the k8s/test directory. * Add the deploy/delete csidriver API in the vineyardctl. * Update the CRD API doc and vineyardctl doc. * Update the helm chart. * Update the base image from gcr.io/distroless/static:nonroot to gcr.io/distroless/base:debug. Signed-off-by: Ye Cao --- .../templates/csidriver-crd.yaml | 115 +++++++ .../templates/deployment.yaml | 2 + .../templates/manager-rbac.yaml | 134 ++++++++ .../templates/recover-crd.yaml | 12 + .../validating-webhook-configuration.yaml | 20 ++ docs/tutorials/kubernetes.rst | 1 + ...ed-on-volumes-with-vineyard-csi-driver.rst | 311 ++++++++++++++++++ k8s/Dockerfile | 3 +- k8s/Makefile | 26 -- k8s/apis/k8s/v1alpha1/README.md | 1 + k8s/apis/k8s/v1alpha1/csidriver_types.go | 6 +- k8s/apis/k8s/v1alpha1/webhook_test.go | 2 +- .../k8s/v1alpha1/zz_generated.deepcopy.go | 38 ++- k8s/cmd/README.md | 114 ++++++- k8s/cmd/commands/csi/csi.go | 4 +- k8s/cmd/commands/delete/delete.go | 1 + k8s/cmd/commands/delete/delete_csidriver.go | 61 ++++ k8s/cmd/commands/deploy/deploy.go | 7 +- k8s/cmd/commands/deploy/deploy_csidriver.go | 107 ++++++ k8s/cmd/commands/flags/csidriver_flags.go | 73 ++++ k8s/cmd/commands/util/parse.go | 21 ++ .../crd/bases/k8s.v6d.io_csidrivers.yaml | 6 +- k8s/examples/vineyard-csidriver/README.md | 205 ------------ .../vineyard-csidriver/prepare-data.yaml | 1 - .../csidriver/Dockerfile} | 0 k8s/test/csidriver/Makefile | 32 ++ 26 files changed, 1057 insertions(+), 246 deletions(-) create mode 100644 charts/vineyard-operator/templates/csidriver-crd.yaml create mode 100644 docs/tutorials/kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst create mode 100644 k8s/cmd/commands/delete/delete_csidriver.go create mode 100644 k8s/cmd/commands/deploy/deploy_csidriver.go create mode 100644 k8s/cmd/commands/flags/csidriver_flags.go delete mode 100644 k8s/examples/vineyard-csidriver/README.md rename k8s/{csitest.Dockerfile => test/csidriver/Dockerfile} (100%) create mode 100644 k8s/test/csidriver/Makefile diff --git a/charts/vineyard-operator/templates/csidriver-crd.yaml b/charts/vineyard-operator/templates/csidriver-crd.yaml new file mode 100644 index 00000000..87ecf588 --- /dev/null +++ b/charts/vineyard-operator/templates/csidriver-crd.yaml @@ -0,0 +1,115 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: csidrivers.k8s.v6d.io + annotations: + cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ include "vineyard-operator.fullname" + . }}-serving-cert' + controller-gen.kubebuilder.io/version: v0.11.0 + labels: + {{- include "vineyard-operator.labels" . | nindent 4 }} +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: webhook-service + namespace: '{{ .Release.Namespace }}' + path: /convert + conversionReviewVersions: + - v1 + group: k8s.v6d.io + names: + kind: CSIDriver + listKind: CSIDriverList + plural: csidrivers + singular: csidriver + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + clusters: + items: + properties: + name: + default: "" + type: string + namespace: + default: "" + type: string + type: object + type: array + enableToleration: + default: false + type: boolean + enableVerboseLog: + default: false + type: boolean + image: + default: vineyardcloudnative/vineyard-operator + type: string + imagePullPolicy: + default: IfNotPresent + type: string + sidecar: + default: + attacherImage: registry.k8s.io/sig-storage/csi-attacher:v4.0.0 + enableTopology: false + imagePullPolicy: Always + livenessProbeImage: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 + nodeRegistrarImage: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0 + provisionerImage: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + properties: + attacherImage: + default: registry.k8s.io/sig-storage/csi-attacher:v4.0.0 + type: string + enableTopology: + default: false + type: boolean + imagePullPolicy: + default: Always + type: string + livenessProbeImage: + default: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 + type: string + nodeRegistrarImage: + default: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0 + type: string + provisionerImage: + default: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + type: string + type: object + storageClassName: + default: vineyard-csi + type: string + volumeBindingMode: + default: WaitForFirstConsumer + type: string + type: object + status: + properties: + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/charts/vineyard-operator/templates/deployment.yaml b/charts/vineyard-operator/templates/deployment.yaml index 3fe861cf..fb77ec81 100644 --- a/charts/vineyard-operator/templates/deployment.yaml +++ b/charts/vineyard-operator/templates/deployment.yaml @@ -11,6 +11,7 @@ metadata: name: {{ include "vineyard-operator.fullname" . }}-controller-manager labels: control-plane: controller-manager + k8s.v6d.io/instance: vineyard-operator {{- include "vineyard-operator.labels" . | nindent 4 }} spec: replicas: {{ .Values.controllerManager.replicas }} @@ -30,6 +31,7 @@ spec: containers: - args: - manager + - --verbose command: - /vineyardctl env: diff --git a/charts/vineyard-operator/templates/manager-rbac.yaml b/charts/vineyard-operator/templates/manager-rbac.yaml index 0c65aa49..f52f5206 100644 --- a/charts/vineyard-operator/templates/manager-rbac.yaml +++ b/charts/vineyard-operator/templates/manager-rbac.yaml @@ -23,6 +23,14 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -34,6 +42,13 @@ rules: - list - patch - update + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/finalizers + verbs: + - patch - apiGroups: - "" resources: @@ -45,6 +60,13 @@ rules: - list - patch - update + - watch +- apiGroups: + - "" + resources: + - persistentvolumes/finalizers + verbs: + - patch - apiGroups: - "" resources: @@ -90,6 +112,16 @@ rules: - delete - get - update +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -127,6 +159,14 @@ rules: - get - list - update +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch - apiGroups: - k8s.v6d.io resources: @@ -147,6 +187,32 @@ rules: - get - patch - update +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers/finalizers + verbs: + - update +- apiGroups: + - k8s.v6d.io + resources: + - csidrivers/status + verbs: + - get + - patch + - update - apiGroups: - k8s.v6d.io resources: @@ -285,6 +351,74 @@ rules: - get - patch - update +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - get + - list + - patch + - update + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/charts/vineyard-operator/templates/recover-crd.yaml b/charts/vineyard-operator/templates/recover-crd.yaml index ff13fe11..ee55908a 100644 --- a/charts/vineyard-operator/templates/recover-crd.yaml +++ b/charts/vineyard-operator/templates/recover-crd.yaml @@ -3,10 +3,22 @@ kind: CustomResourceDefinition metadata: name: recovers.k8s.v6d.io annotations: + cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ include "vineyard-operator.fullname" + . }}-serving-cert' controller-gen.kubebuilder.io/version: v0.11.0 labels: {{- include "vineyard-operator.labels" . | nindent 4 }} spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: webhook-service + namespace: '{{ .Release.Namespace }}' + path: /convert + conversionReviewVersions: + - v1 group: k8s.v6d.io names: kind: Recover diff --git a/charts/vineyard-operator/templates/validating-webhook-configuration.yaml b/charts/vineyard-operator/templates/validating-webhook-configuration.yaml index b8d0cbb2..ff4e2687 100644 --- a/charts/vineyard-operator/templates/validating-webhook-configuration.yaml +++ b/charts/vineyard-operator/templates/validating-webhook-configuration.yaml @@ -27,6 +27,26 @@ webhooks: resources: - backups sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ include "vineyard-operator.fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /validate-k8s-v6d-io-v1alpha1-csidriver + failurePolicy: Fail + name: vcsidriver.kb.io + rules: + - apiGroups: + - k8s.v6d.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - csidrivers + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/docs/tutorials/kubernetes.rst b/docs/tutorials/kubernetes.rst index d689e590..a4d84ed9 100644 --- a/docs/tutorials/kubernetes.rst +++ b/docs/tutorials/kubernetes.rst @@ -9,6 +9,7 @@ Vineyard on Kubernetes ./kubernetes/using-vineyard-operator.rst ./kubernetes/ml-pipeline-mars-pytorch.rst ./kubernetes/data-sharing-with-vineyard-on-kubernetes.rst + ./kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst Vineyard can be seamlessly deployed on Kubernetes, managed by the :ref:`vineyard-operator`, to enhance big-data workflows through its data-aware scheduling policy. This policy diff --git a/docs/tutorials/kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst b/docs/tutorials/kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst new file mode 100644 index 00000000..7ffb65d4 --- /dev/null +++ b/docs/tutorials/kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst @@ -0,0 +1,311 @@ +Speed up pipelines based on volumes with Vineyard CSI Driver +============================================================ + +If you are using `Kubeflow Pipeline`_ or `Argo Workflow`_ to manage your machine learning workflow, +you may find that the data saving/loading to the volumes is slow. +To speed up the data saving/loading within these volumes, we design the Vineyard CSI Driver to +map each vineyard object to a volume, and the data saving/loading is handled by vineyard. +Next, we will show you how to use the Vineyard CSI Driver to speed up a kubeflow pipeline. + +Prerequisites +------------- + +- A kubernetes cluster with version >= 1.25.10. If you don't have one by hand, you can refer to the +guide `Prepare the Kubernetes Cluster`_ to create one. +- Install the `Vineyardctl`_ by following the official guide. +- Install the `Argo Workflow CLI`_ by following the official guide. + +Deploy the Vineyard Cluster +--------------------------- + +.. code:: bash + + $ vineyardctl deploy vineyard-cluster --create-namespace + +This command will create a vineyard cluster in the namespace `vineyard-system`. +You can check as follows: + +.. code:: bash + + $ kubectl get pod -n vineyard-system + NAME READY STATUS RESTARTS AGE + vineyard-controller-manager-648fc9b7bf-zwnhd 2/2 Running 0 4d3h + vineyardd-sample-79c8ffb879-6k8mk 1/1 Running 0 4d3h + vineyardd-sample-79c8ffb879-f9kkr 1/1 Running 0 4d3h + vineyardd-sample-79c8ffb879-lzgwz 1/1 Running 0 4d3h + vineyardd-sample-etcd-0 1/1 Running 0 4d3h + +Deploy the Vineyard CSI Driver +------------------------------ + +Before deploying the Vineyard CSI Driver, you are supposed to check the vineyard +deployment is ready as follows: + +.. code:: bash + + $ kubectl get deployment -n vineyard-system + NAME READY UP-TO-DATE AVAILABLE AGE + vineyard-controller-manager 1/1 1 1 4d3h + vineyardd-sample 3/3 3 3 4d3h + +Then deploy the vineyard csi driver which specifies the vineyard cluster to use: + +.. code:: bash + + $ vineyardctl deploy csidriver --clusters vineyard-system/vineyardd-sample + +Then check the status of the Vineyard CSI Driver: + +.. code:: bash + + $ kubectl get pod -n vineyard-system + NAME READY STATUS RESTARTS AGE + vineyard-controller-manager-648fc9b7bf-zwnhd 2/2 Running 0 4d3h + vineyard-csi-sample-csi-driver-fb7cb5b5d-nlrxs 4/4 Running 0 4m23s + vineyard-csi-sample-csi-nodes-69j77 3/3 Running 0 4m23s + vineyard-csi-sample-csi-nodes-k85hb 3/3 Running 0 4m23s + vineyard-csi-sample-csi-nodes-zhfz4 3/3 Running 0 4m23s + vineyardd-sample-79c8ffb879-6k8mk 1/1 Running 0 4d3h + vineyardd-sample-79c8ffb879-f9kkr 1/1 Running 0 4d3h + vineyardd-sample-79c8ffb879-lzgwz 1/1 Running 0 4d3h + vineyardd-sample-etcd-0 1/1 Running 0 4d3h + +Deploy Argo Workflows +--------------------- + +Install the argo server on Kubernetes: + +.. code:: bash + + $ kubectl create namespace argo + $ kubectl apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.4.8/install.yaml + +Then check the status of the argo server: + +.. code:: bash + + $ kubectl get pod -n argo + NAME READY STATUS RESTARTS AGE + argo-server-7698c96655-ft6sj 1/1 Running 0 4d1h + workflow-controller-b888f4458-sfrjd 1/1 Running 0 4d1h + +Running a Kubeflow Pipeline example +----------------------------------- + +The example is under the directory `k8s/examples/vineyard-csidriver`, and `pipeline.py` under this +directory is the original pipeline definition. To use the Vineyard CSI Driver, we need to do two +modifications: + +1. Change APIs like `pd.read_pickle/write_pickle` to `vineyard.csi.write/read` in the source code. + +2. Add the `vineyard object` VolumeOp to the pipeline's dependencies. The path in the API changed +in the first step will be mapped to a volume. Notice, the volume used in any task needs to be +explicitly mounted to the corresponding path in the source code, and the storageclass_name +format of each VolumeOp is `{vineyard-deployment-namespace}.{vineyard-deployment-name}.csi`. + +You may get some insights from the modified pipeline `pipeline-with-vineyard.py`. Then, we need to +compile the pipeline to an argo-workflow yaml. To be compatible with benchmark test, we update the +generated `pipeline.yaml` and `pipeline-with-vineyard.yaml`. + +Now, we can build the docker images for the pipeline: + +.. code:: bash + + $ cd k8s/examples/vineyard-csidriver + $ make docker-build + +Check the images built successfully: + +.. code:: bash + + $ docker images + train-data latest 5628953ffe08 14 seconds ago 1.47GB + test-data latest 94c8c75b960a 14 seconds ago 1.47GB + prepare-data latest 5aab1b120261 15 seconds ago 1.47GB + preprocess-data latest 5246d09e6f5e 15 seconds ago 1.47GB + +Then push the image to a docker registry that your kubernetes cluster can access, as +we use the kind cluster in this example, we can load the image to the clusters: + +.. code:: bash + + $ make load-images + +To simulate the data loading/saving of the actual pipeline, we use the nfs volume +to store the data. The nfs volume is mounted to the `/mnt/data` directory of the +kind cluster. Then apply the data volume as follows: + +.. tip:: + + If you already have nfs volume that can be accessed by the kubernetes cluster, + you can update the prepare-data.yaml to use your nfs volume. + +.. code:: bash + + $ kubectl apply -f prepare-data.yaml + +Deploy the rbac for the pipeline: + +.. code:: bash + + $ kubectl apply -f rbac.yaml + +Submit the kubeflow example without vineyard to the argo server: + +.. code:: bash + + $ for data_multiplier in 3000 4000 5000; do \ + argo submit --watch pipeline.yaml -p data_multiplier=${data_multiplier}; \ + done + +Clear the previous resources: + +.. code:: bash + + $ argo delete --all + +Submit the kubeflow example with vineyard to the argo server: + +.. code:: bash + + $ for data_multiplier in 3000 4000 5000; do \ + argo submit --watch pipeline-with-vineyard.yaml -p data_multiplier=${data_multiplier}; \ + done + +Result Analysis +--------------- + ++------------+------------------+---------------+ +| data scale | without vineyard | with vineyard | ++============+==================+===============+ +| 8500 Mi | 21s | 5.4s | +| 12000 Mi | 26s | 7s | +| 15000 Mi | 32.2s | 9.4s | ++------------+------------------+---------------+ + +The data scale are 8500 Mi, 12000 Mi and 15000 Mi, which correspond to +the 3000, 4000 and 5000 in the previous data_multiplier respectively, +and the time of argo workflow execution of the pipeline is as follows: + +Argo workflow duration +====================== + ++------------+------------------+---------------+ +| data scale | without vineyard | with vineyard | ++============+==================+===============+ +| 8500 Mi | 186s | 169s | +| 12000 Mi | 250s | 203s | +| 15000 Mi | 332s | 286s | ++------------+------------------+---------------+ + + +Actually, the cost time of argo workflow is affected by lots of factors, +such as the network, the cpu and memory of the cluster, the data volume, etc. +So the time of argo workflow execution of the pipeline is not stable. +But we can still find that the time of argo workflow execution of the pipeline +with vineyard is shorter than that without vineyard. + +Also, we record the whole execution time via logs. The result is as follows: + +Actual execution time +===================== + ++------------+------------------+---------------+ +| data scale | without vineyard | with vineyard | ++============+==================+===============+ +| 8500 Mi | 139.3s | 92.3s | +| 12000 Mi | 204.3s | 131.1s | +| 15000 Mi | 289.3s | 209.7s | ++------------+------------------+---------------+ + + +According to the above results, we can find that the time of actual +execution of the pipeline with vineyard is shorter than that without vineyard. +To be specific, we record the write/read time of the following steps: + +Writing time +============ + ++------------+------------------+---------------+ +| data scale | without vineyard | with vineyard | ++============+==================+===============+ +| 8500 Mi | 21s | 5.4s | +| 12000 Mi | 26s | 7s | +| 15000 Mi | 32.2s | 9.4s | ++------------+------------------+---------------+ + + +From the above results, we can find that the writing time the pipeline +with vineyard is nearly 4 times shorter than that without vineyard. +The reason is that the data is stored in the vineyard cluster, +so it's actually a memory copy operation, which is faster than the +write operation of the nfs volume. + + +Reading time +============ + +We delete the time of init data loading, and the results are as follows: + ++------------+------------------+---------------+ +| data scale | without vineyard | with vineyard | ++============+==================+===============+ +| 8500 Mi | 36.7s | 0.02s | +| 12000 Mi | 45.7s | 0.02s | +| 15000 Mi | 128.6s | 0.04s | ++------------+------------------+---------------+ + +Based on the above results, we can find that the read time of vineyard is +nearly a constant, which is not affected by the data scale. +The reason is that the data is stored in the shared memory of vineyard cluster, +so it's actually a pointer copy operation. + +As a result, we can find that with vineyard, the argo workflow +duration of the pipeline is reduced by 10%~20% and the actual +execution time of the pipeline is reduced by about 30%. + + +Clean up +-------- + +Delete the rbac for the kubeflow example: + +.. code:: bash + + $ kubectl delete -f rbac.yaml + +Delete all argo workflow + +.. code:: bash + + $ argo delete --all + +Delete the argo server: + +.. code:: bash + + $ kubectl delete ns argo + +Delete the csi driver: + +.. code:: bash + + $ vineyardctl delete csidriver + +Delete the vineyard cluster: + +.. code:: bash + + $ vineyardctl delete vineyard-cluster + +Delete the data volume: + +.. code:: bash + + $ kubectl delete -f prepare-data.yaml + +.. _Kubeflow Pipeline: https://github.com/kubeflow/kubeflow +.. _Argo Workflow: https://github.com/argoproj/argo-workflows +.. _Prepare the Kubernetes Cluster: https://v6d.io/tutorials/data-processing/accelerate-data-sharing-in-kedro.html#prepare-the-kubernetes-cluster +.. _Vineyardctl: https://v6d.io/notes/developers/build-from-source.html#install-vineyardctl +.. _Argo Workflow CLI: https://github.com/argoproj/argo-workflows/releases/ \ No newline at end of file diff --git a/k8s/Dockerfile b/k8s/Dockerfile index c0438492..a67504e5 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -1,4 +1,3 @@ -ARG BASE_IMAGE=gcr.io/distroless/static:nonroot # Build the manager binary FROM golang:1.19 as builder @@ -25,7 +24,7 @@ RUN go build -a -o vineyardctl cmd/main.go && \ # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM ${BASE_IMAGE} +FROM gcr.io/distroless/base:debug WORKDIR / COPY k8s/config/scheduler/config.yaml /etc/kubernetes/scheduler.yaml diff --git a/k8s/Makefile b/k8s/Makefile index 17a44824..3c71839c 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -30,7 +30,6 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) REGISTRY := vineyardcloudnative IMG ?= $(REGISTRY)/vineyard-operator:$(VERSION) VINEYARD_CSI_IMAGE ?= $(REGISTRY)/vineyard-csi-driver:$(VERSION) -VINEYARD_CSI_TEST_IMAGE ?= $(REGISTRY)/vineyard-csi-driver-test:$(VERSION) temp=$(shell mktemp -d) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -232,31 +231,6 @@ docker-build: -t $(IMG); \ fi -.PHONY: csidriver -csidriver: - cd .. && \ - if docker build --help | grep -q load; then \ - docker build --load -f k8s/Dockerfile . \ - --build-arg BASE_IMAGE=gcr.io/distroless/base:debug \ - -t $(VINEYARD_CSI_IMAGE); \ - else \ - docker build -f k8s/Dockerfile . \ - --build-arg BASE_IMAGE=gcr.io/distroless/base:debug \ - -t $(VINEYARD_CSI_IMAGE); \ - fi - -.PHONY: csidriver-test -csidriver-test: - cd .. && \ - if docker build --help | grep -q load; then \ - docker build --load -f k8s/csitest.Dockerfile . \ - -t $(VINEYARD_CSI_TEST_IMAGE); \ - else \ - docker build -f k8s/csitest.Dockerfile . \ - -t $(VINEYARD_CSI_TEST_IMAGE); \ - fi && \ - docker run --rm -it --privileged=true $(VINEYARD_CSI_TEST_IMAGE) - docker-build-push-multi-arch: cd .. && \ docker buildx build -f k8s/Dockerfile . -t $(IMG) --platform linux/amd64,linux/arm64 --push diff --git a/k8s/apis/k8s/v1alpha1/README.md b/k8s/apis/k8s/v1alpha1/README.md index 329a2e4d..7e8d6410 100644 --- a/k8s/apis/k8s/v1alpha1/README.md +++ b/k8s/apis/k8s/v1alpha1/README.md @@ -133,6 +133,7 @@ _Appears in:_ | `sidecar` _[CSISidecar](#csisidecar)_ | Sidecar is the configuration for the CSI sidecar container nolint: lll | | `clusters` _[VineyardClusters](#vineyardclusters) array_ | Clusters are the list of vineyard clusters | | `enableToleration` _boolean_ | EnableToleration is the flag to enable toleration for the csi driver | +| `enableVerboseLog` _boolean_ | EnableVerboseLog is the flag to enable verbose log for the csi driver | diff --git a/k8s/apis/k8s/v1alpha1/csidriver_types.go b/k8s/apis/k8s/v1alpha1/csidriver_types.go index 3c57679b..ab10d80f 100644 --- a/k8s/apis/k8s/v1alpha1/csidriver_types.go +++ b/k8s/apis/k8s/v1alpha1/csidriver_types.go @@ -69,7 +69,7 @@ type VineyardClusters struct { type CSIDriverSpec struct { // Image is the name of the csi driver image // +kubebuilder:validation:Required - // +kubebuilder:default:="vineyardcloudnative/vineyard-csi-driver" + // +kubebuilder:default:="vineyardcloudnative/vineyard-operator" Image string `json:"image,omitempty"` // ImagePullPolicy is the image pull policy of the csi driver @@ -103,10 +103,10 @@ type CSIDriverSpec struct { // +kubebuilder:default:=false EnableToleration bool `json:"enableToleration,omitempty"` - // EnableDebugLog is the flag to enable debug log for the csi driver + // EnableVerboseLog is the flag to enable verbose log for the csi driver // +kubebuilder:validation:Required // +kubebuilder:default:=false - EnableDebugLog bool `json:"enableDebugLog,omitempty"` + EnableVerboseLog bool `json:"enableVerboseLog,omitempty"` } // CSIDriverStatus defines the observed state of CSIDriver diff --git a/k8s/apis/k8s/v1alpha1/webhook_test.go b/k8s/apis/k8s/v1alpha1/webhook_test.go index 74b7027d..2d741af8 100644 --- a/k8s/apis/k8s/v1alpha1/webhook_test.go +++ b/k8s/apis/k8s/v1alpha1/webhook_test.go @@ -111,7 +111,7 @@ func Test_webhook(t *testing.T) { assert.NoError(t, err) err = (&CSIDriver{}).SetupWebhookWithManager(mgr) - Expect(err).NotTo(HaveOccurred()) + assert.NoError(t, err) //+kubebuilder:scaffold:webhook diff --git a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go index ce8e2277..7a482e8b 100644 --- a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go +++ b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go @@ -127,7 +127,7 @@ func (in *CSIDriver) DeepCopyInto(out *CSIDriver) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status } @@ -184,6 +184,12 @@ func (in *CSIDriverList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) { *out = *in + out.Sidecar = in.Sidecar + if in.Clusters != nil { + in, out := &in.Clusters, &out.Clusters + *out = make([]VineyardClusters, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverSpec. @@ -211,6 +217,21 @@ func (in *CSIDriverStatus) DeepCopy() *CSIDriverStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSISidecar) DeepCopyInto(out *CSISidecar) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSISidecar. +func (in *CSISidecar) DeepCopy() *CSISidecar { + if in == nil { + return nil + } + out := new(CSISidecar) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalObject) DeepCopyInto(out *GlobalObject) { *out = *in @@ -736,6 +757,21 @@ func (in *SpillConfig) DeepCopy() *SpillConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VineyardClusters) DeepCopyInto(out *VineyardClusters) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VineyardClusters. +func (in *VineyardClusters) DeepCopy() *VineyardClusters { + if in == nil { + return nil + } + out := new(VineyardClusters) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VineyardConfig) DeepCopyInto(out *VineyardConfig) { *out = *in diff --git a/k8s/cmd/README.md b/k8s/cmd/README.md index 7df513f5..a8b81240 100644 --- a/k8s/cmd/README.md +++ b/k8s/cmd/README.md @@ -13,6 +13,7 @@ drivers. **SEE ALSO** * [vineyardctl create](#vineyardctl-create) - Create a vineyard jobs on kubernetes +* [vineyardctl csi](#vineyardctl-csi) - Start the vineyard csi driver * [vineyardctl delete](#vineyardctl-delete) - Delete the vineyard components from kubernetes * [vineyardctl deploy](#vineyardctl-deploy) - Deploy the vineyard components on kubernetes * [vineyardctl get](#vineyardctl-get) - Get vineyard object, metadata, blob or cluster-info @@ -270,6 +271,34 @@ vineyardctl create recover [flags] --recover-name string the name of recover job (default "vineyard-recover") ``` +## `vineyardctl csi` + +Start the vineyard csi driver + +``` +vineyardctl csi [flags] +``` + +**SEE ALSO** + +* [vineyardctl](#vineyardctl) - vineyardctl is the command-line tool for interact with the Vineyard Operator. + +### Examples + +```shell + # start the csi with the specific endpoint and node id + vineyardctl csi --endpoint=unix:///csi/csi.sock --nodeid=csinode1 +``` + +### Options + +``` + -f, --endpoint string the endpoint of vineyard csi driver + -h, --help help for csi + --nodeid string the node id of vineyard csi driver + --state-file-path string the path of state file (default "/csi/state") +``` + ## `vineyardctl delete` Delete the vineyard components from kubernetes @@ -279,6 +308,7 @@ Delete the vineyard components from kubernetes * [vineyardctl](#vineyardctl) - vineyardctl is the command-line tool for interact with the Vineyard Operator. * [vineyardctl delete backup](#vineyardctl-delete-backup) - Delete the backup job on kubernetes * [vineyardctl delete cert-manager](#vineyardctl-delete-cert-manager) - Delete the cert-manager on kubernetes +* [vineyardctl delete csidriver](#vineyardctl-delete-csidriver) - Delete the vineyard csi driver on kubernetes * [vineyardctl delete operation](#vineyardctl-delete-operation) - Delete the operation from kubernetes * [vineyardctl delete operator](#vineyardctl-delete-operator) - Delete the vineyard operator from kubernetes * [vineyardctl delete recover](#vineyardctl-delete-recover) - Delete the recover job from kubernetes @@ -368,6 +398,32 @@ vineyardctl delete cert-manager [flags] -h, --help help for cert-manager ``` +## `vineyardctl delete csidriver` + +Delete the vineyard csi driver on kubernetes + +``` +vineyardctl delete csidriver [flags] +``` + +**SEE ALSO** + +* [vineyardctl delete](#vineyardctl-delete) - Delete the vineyard components from kubernetes + +### Examples + +```shell + # delete the csi driver named "csidriver-test" + vineyardctl delete csidriver --name csidriver-test +``` + +### Options + +``` + -h, --help help for csidriver + --name string The name of the csi driver cr. (default "csidriver-sample") +``` + ## `vineyardctl delete operation` Delete the operation from kubernetes @@ -541,6 +597,7 @@ Deploy the vineyard components on kubernetes * [vineyardctl](#vineyardctl) - vineyardctl is the command-line tool for interact with the Vineyard Operator. * [vineyardctl deploy backup-job](#vineyardctl-deploy-backup-job) - Deploy a backup job of vineyard cluster on kubernetes * [vineyardctl deploy cert-manager](#vineyardctl-deploy-cert-manager) - Deploy the cert-manager on kubernetes +* [vineyardctl deploy csidriver](#vineyardctl-deploy-csidriver) - Deploy the vineyard csi driver on kubernetes * [vineyardctl deploy operator](#vineyardctl-deploy-operator) - Deploy the vineyard operator on kubernetes * [vineyardctl deploy recover-job](#vineyardctl-deploy-recover-job) - Deploy a recover job to recover a backup of current vineyard cluster on kubernetes * [vineyardctl deploy vineyard-cluster](#vineyardctl-deploy-vineyard-cluster) - Deploy the vineyard cluster from kubernetes @@ -561,6 +618,10 @@ Deploy the vineyard components on kubernetes # deploy the vineyardd on kubernetes vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy vineyardd + + # deploy the vineyard csi driver on kubernetes + vineyardctl deploy csidriver --name vineyard-csi-sample \ + --clusters vineyard-system/vineyardd-sample,default/vineyardd-sample ``` ### Options @@ -729,6 +790,53 @@ vineyardctl deploy cert-manager [flags] -h, --help help for cert-manager ``` +## `vineyardctl deploy csidriver` + +Deploy the vineyard csi driver on kubernetes + +### Synopsis + +Deploy the Vineyard CSI Driver on kubernetes. +The CR is a cluster-scoped resource, and can only be created once. + +``` +vineyardctl deploy csidriver [flags] +``` + +**SEE ALSO** + +* [vineyardctl deploy](#vineyardctl-deploy) - Deploy the vineyard components on kubernetes + +### Examples + +```shell + # deploy the Vineyard CSI Driver named vineyard-csi-sample on kubernetes + # Notice, the clusters are built as {vineyard-deployment-namespace}/{vineyard-deployment-name} + # and sperated by comma, e.g. vineyard-system/vineyardd-sample, default/vineyardd-sample + # They must be created before deploying the Vineyard CSI Driver. + vineyardctl deploy csidriver --name vineyard-csi-sample \ + --clusters vineyard-system/vineyardd-sample,default/vineyardd-sample +``` + +### Options + +``` + --attacherImage string The image of csi attacher. (default "registry.k8s.io/sig-storage/csi-attacher:v4.0.0") + --clusters strings The list of vineyard clusters. + --enableToleration Enable toleration for vineyard csi driver. + -h, --help help for csidriver + -i, --image string The image of vineyard csi driver. (default "vineyardcloudnative/vineyard-csi-driver") + --imagePullPolicy string The image pull policy of vineyard csi driver. (default "IfNotPresent") + --livenessProbeImage string The image of livenessProbe. (default "registry.k8s.io/sig-storage/livenessprobe:v2.8.0") + --name string The name of the csi driver cr. (default "csidriver-sample") + --nodeRegistrarImage string The image of csi nodeRegistrar. (default "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0") + --provisionerImage string The image of csi provisioner. (default "registry.k8s.io/sig-storage/csi-provisioner:v3.3.0") + --sidecar.enableTopology Enable topology for the csi driver. + --sidecar.imagePullPolicy string The image pull policy of all sidecar containers. (default "Always") + -s, --storageClassName string The name of storage class. (default "vineyard-csi") + -m, --volumeBindingMode string The volume binding mode of the storage class. (default "WaitForFirstConsumer") +``` + ## `vineyardctl deploy operator` Deploy the vineyard operator on kubernetes @@ -2160,4 +2268,8 @@ vineyardctl schedule workload [flags] --resource string the json string of kubernetes workload --vineyardd-name string the namespace of vineyard cluster (default "vineyardd-sample") --vineyardd-namespace string the namespace of vineyard cluster (default "vineyard-system") -``` \ No newline at end of file +``` + +stem") +``` + diff --git a/k8s/cmd/commands/csi/csi.go b/k8s/cmd/commands/csi/csi.go index b0afc46c..82e12904 100644 --- a/k8s/cmd/commands/csi/csi.go +++ b/k8s/cmd/commands/csi/csi.go @@ -26,8 +26,8 @@ import ( ) var csiExample = util.Examples(` - # start the csidriver with the specific endpoint and node id - vineyardctl csidriver --endpoint=unix:///csi/csi.sock --nodeid=csinode1`) + # start the csi with the specific endpoint and node id + vineyardctl csi --endpoint=unix:///csi/csi.sock --nodeid=csinode1`) // csiCmd starts the vineyard csi driver var csiCmd = &cobra.Command{ diff --git a/k8s/cmd/commands/delete/delete.go b/k8s/cmd/commands/delete/delete.go index 20caa628..9f3eaa71 100644 --- a/k8s/cmd/commands/delete/delete.go +++ b/k8s/cmd/commands/delete/delete.go @@ -55,4 +55,5 @@ func init() { deleteCmd.AddCommand(NewDeleteRecoverCmd()) deleteCmd.AddCommand(NewDeleteVineyardDeploymentCmd()) deleteCmd.AddCommand(NewDeleteOperationCmd()) + deleteCmd.AddCommand(NewDeleteCSIDriverCmd()) } diff --git a/k8s/cmd/commands/delete/delete_csidriver.go b/k8s/cmd/commands/delete/delete_csidriver.go new file mode 100644 index 00000000..aee2753c --- /dev/null +++ b/k8s/cmd/commands/delete/delete_csidriver.go @@ -0,0 +1,61 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package delete + +import ( + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/types" + + "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1" + "github.com/v6d-io/v6d/k8s/cmd/commands/flags" + "github.com/v6d-io/v6d/k8s/cmd/commands/util" + "github.com/v6d-io/v6d/k8s/pkg/log" +) + +var deleteCSIDriverExample = util.Examples(` + # delete the csi driver named "csidriver-test" + vineyardctl delete csidriver --name csidriver-test`) + +// deleteCSIDriverCmd deletes the specific operation +var deleteCSIDriverCmd = &cobra.Command{ + Use: "csidriver", + Short: "Delete the vineyard csi driver on kubernetes", + Example: deleteCSIDriverExample, + Run: func(cmd *cobra.Command, args []string) { + util.AssertNoArgs(cmd, args) + + client := util.KubernetesClient() + + log.Info("deleting CSIDriver cr") + csiDriver := &v1alpha1.CSIDriver{} + + if err := util.Delete(client, types.NamespacedName{ + Name: flags.CSIDriverName, + }, csiDriver); err != nil { + log.Fatal(err, "failed to delete CSIDriver") + } + + log.Info("CSIDriver is deleted.") + }, +} + +func NewDeleteCSIDriverCmd() *cobra.Command { + return deleteCSIDriverCmd +} + +func init() { + flags.ApplyCSIDriverNameOpts(deleteCSIDriverCmd) +} diff --git a/k8s/cmd/commands/deploy/deploy.go b/k8s/cmd/commands/deploy/deploy.go index 32816482..580e096a 100644 --- a/k8s/cmd/commands/deploy/deploy.go +++ b/k8s/cmd/commands/deploy/deploy.go @@ -32,7 +32,11 @@ var deployExample = util.Examples(` vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy cert-manager # deploy the vineyardd on kubernetes - vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy vineyardd`) + vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy vineyardd + + # deploy the vineyard csi driver on kubernetes + vineyardctl deploy csidriver --name vineyard-csi-sample \ + --clusters vineyard-system/vineyardd-sample,default/vineyardd-sample`) // deployCmd deploys all vineyard components on kubernetes var deployCmd = &cobra.Command{ @@ -53,4 +57,5 @@ func init() { deployCmd.AddCommand(NewDeployVineyardDeploymentCmd()) deployCmd.AddCommand(NewDeployBackupJobCmd()) deployCmd.AddCommand(NewDeployRecoverJobCmd()) + deployCmd.AddCommand(NewDeployCSIDriverCmd()) } diff --git a/k8s/cmd/commands/deploy/deploy_csidriver.go b/k8s/cmd/commands/deploy/deploy_csidriver.go new file mode 100644 index 00000000..0c995e9f --- /dev/null +++ b/k8s/cmd/commands/deploy/deploy_csidriver.go @@ -0,0 +1,107 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package deploy + +import ( + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1" + "github.com/v6d-io/v6d/k8s/cmd/commands/flags" + "github.com/v6d-io/v6d/k8s/cmd/commands/util" + "github.com/v6d-io/v6d/k8s/pkg/log" +) + +var ( + deployCSIDriverLong = util.LongDesc(` + Deploy the Vineyard CSI Driver on kubernetes. + The CR is a cluster-scoped resource, and can only be created once.`) + + deployCSIDriverExample = util.Examples(` + # deploy the Vineyard CSI Driver named vineyard-csi-sample on kubernetes + # Notice, the clusters are built as {vineyard-deployment-namespace}/{vineyard-deployment-name} + # and sperated by comma, e.g. vineyard-system/vineyardd-sample, default/vineyardd-sample + # They must be created before deploying the Vineyard CSI Driver. + vineyardctl deploy csidriver --name vineyard-csi-sample \ + --clusters vineyard-system/vineyardd-sample,default/vineyardd-sample`) +) + +// deployCSIDriverCmd deploys the vineyard csi driver on kubernetes +var deployCSIDriverCmd = &cobra.Command{ + Use: "csidriver", + Short: "Deploy the vineyard csi driver on kubernetes", + Long: deployCSIDriverLong, + Example: deployCSIDriverExample, + Run: func(cmd *cobra.Command, args []string) { + util.AssertNoArgsOrInput(cmd, args) + + client := util.KubernetesClient() + log.Info("building CSIDriver cr") + csiDriver, err := BuildCSIDriver() + if err != nil { + log.Fatal(err, "Failed to build csi driver") + } + log.Info("creating csi driver") + if err := util.Create(client, csiDriver, func(csiDriver *v1alpha1.CSIDriver) bool { + return csiDriver.Status.State == v1alpha1.CSIDriverRunning + }); err != nil { + log.Fatal(err, "failed to create/wait Vineyard CSI Driver") + } + + log.Info("Vineyard CSI Driver is ready.") + }, +} + +func checkVolumeBindingMode(mode string) bool { + switch mode { + case "WaitForFirstConsumer", "Immediate": + return true + default: + return false + } +} + +func BuildCSIDriver() (*v1alpha1.CSIDriver, error) { + clusters, err := util.ParseVineyardClusters(flags.VineyardClusters) + if err != nil { + return nil, err + } + csiDriver := &v1alpha1.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: flags.CSIDriverName, + Namespace: flags.GetDefaultVineyardNamespace(), + }, + Spec: flags.CSIDriverOpts, + } + if !checkVolumeBindingMode(csiDriver.Spec.VolumeBindingMode) { + return nil, fmt.Errorf("invalid volume binding mode: %s, "+ + "only support WaitForFirstConsumer and Immediate", + csiDriver.Spec.VolumeBindingMode) + } + csiDriver.Spec.Clusters = *clusters + csiDriver.Spec.EnableVerboseLog = flags.Verbose + return csiDriver, nil +} + +func NewDeployCSIDriverCmd() *cobra.Command { + return deployCSIDriverCmd +} + +func init() { + flags.ApplyCSIDriverOpts(deployCSIDriverCmd) +} diff --git a/k8s/cmd/commands/flags/csidriver_flags.go b/k8s/cmd/commands/flags/csidriver_flags.go new file mode 100644 index 00000000..5b76fb7b --- /dev/null +++ b/k8s/cmd/commands/flags/csidriver_flags.go @@ -0,0 +1,73 @@ +/* +* Copyright 2020-2023 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package flags + +import ( + "github.com/spf13/cobra" + "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1" +) + +var ( + // CSIDriverOpts holds all configuration of CSIDriver Spec + CSIDriverOpts v1alpha1.CSIDriverSpec + + // CSIDriverName is the name of the csi driver cr + CSIDriverName string + + // VineyardClusters contains all the vineyard clusters + VineyardClusters []string +) + +func ApplyCSIDriverNameOpts(cmd *cobra.Command) { + cmd.Flags().StringVarP(&CSIDriverName, "name", "", "csidriver-sample", + "The name of the csi driver cr.") +} +func ApplyCSIDriverSidecarOpts(cmd *cobra.Command) { + cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.ProvisionerImage, "provisionerImage", "", + "registry.k8s.io/sig-storage/csi-provisioner:v3.3.0", "The image of csi provisioner.") + cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.AttacherImage, "attacherImage", "", + "registry.k8s.io/sig-storage/csi-attacher:v4.0.0", "The image of csi attacher.") + cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.NodeRegistrarImage, "nodeRegistrarImage", "", + "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0", "The image of csi nodeRegistrar.") + cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.LivenessProbeImage, "livenessProbeImage", "", + "registry.k8s.io/sig-storage/livenessprobe:v2.8.0", "The image of livenessProbe.") + cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.ImagePullPolicy, "sidecar.imagePullPolicy", "", + "Always", "The image pull policy of all sidecar containers.") + cmd.Flags().BoolVarP(&CSIDriverOpts.Sidecar.EnableTopology, "sidecar.enableTopology", "", + false, "Enable topology for the csi driver.") +} + +func ApplyCSIDriverClustersOpts(cmd *cobra.Command) { + cmd.Flags().StringSliceVarP(&VineyardClusters, "clusters", "", + []string{}, "The list of vineyard clusters.") +} + +func ApplyCSIDriverOpts(cmd *cobra.Command) { + ApplyCSIDriverNameOpts(cmd) + cmd.Flags().StringVarP(&CSIDriverOpts.Image, "image", "i", + "vineyardcloudnative/vineyard-csi-driver", + "The image of vineyard csi driver.") + cmd.Flags().StringVarP(&CSIDriverOpts.ImagePullPolicy, "imagePullPolicy", "", + "IfNotPresent", "The image pull policy of vineyard csi driver.") + cmd.Flags().StringVarP(&CSIDriverOpts.StorageClassName, "storageClassName", "s", + "vineyard-csi", "The name of storage class.") + cmd.Flags().StringVarP(&CSIDriverOpts.VolumeBindingMode, "volumeBindingMode", "m", + "WaitForFirstConsumer", "The volume binding mode of the storage class.") + cmd.Flags().BoolVarP(&CSIDriverOpts.EnableToleration, "enableToleration", "", + false, "Enable toleration for vineyard csi driver.") + ApplyCSIDriverSidecarOpts(cmd) + ApplyCSIDriverClustersOpts(cmd) +} diff --git a/k8s/cmd/commands/util/parse.go b/k8s/cmd/commands/util/parse.go index 33aee0c9..200f27ce 100644 --- a/k8s/cmd/commands/util/parse.go +++ b/k8s/cmd/commands/util/parse.go @@ -17,9 +17,11 @@ package util import ( "encoding/json" + "fmt" "strings" "github.com/ghodss/yaml" + "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1" "github.com/pkg/errors" @@ -180,3 +182,22 @@ func ParseOwnerRef(of string) ([]metav1.OwnerReference, error) { return ownerRef, nil } + +// ParseVineyardClusters parse the []string to nested []{ +// "namespace": "vineyard-cluster-namespace", +// "name": "vineyard-cluster-name", +// } +func ParseVineyardClusters(clusters []string) (*[]v1alpha1.VineyardClusters, error) { + vineyardClusters := make([]v1alpha1.VineyardClusters, 0) + for i := range clusters { + s := strings.Split(clusters[i], "/") + if len(s) != 2 { + return nil, errors.Wrap(fmt.Errorf("invalid vineyard cluster %s", clusters[i]), "parse vineyard cluster") + } + vineyardClusters = append(vineyardClusters, v1alpha1.VineyardClusters{ + Namespace: s[0], + Name: s[1], + }) + } + return &vineyardClusters, nil +} diff --git a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml index e1b8e243..c74d2cc3 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml @@ -38,14 +38,14 @@ spec: type: string type: object type: array - enableDebugLog: + enableToleration: default: false type: boolean - enableToleration: + enableVerboseLog: default: false type: boolean image: - default: vineyardcloudnative/vineyard-csi-driver + default: vineyardcloudnative/vineyard-operator type: string imagePullPolicy: default: IfNotPresent diff --git a/k8s/examples/vineyard-csidriver/README.md b/k8s/examples/vineyard-csidriver/README.md deleted file mode 100644 index 61989c0a..00000000 --- a/k8s/examples/vineyard-csidriver/README.md +++ /dev/null @@ -1,205 +0,0 @@ -# Kubeflow Example with Vineyard CSI Driver - -## Prerequisites - -If you don't have a kubernetes cluster by hand, you can use [kind](https://kind.sigs.k8s.io/) to create a kubernetes cluster: - -```bash -$ cat < Date: Mon, 18 Sep 2023 19:19:24 +0800 Subject: [PATCH 4/4] Update the title of the vineyard csi driver tutorial. Signed-off-by: Ye Cao --- docs/tutorials/kubernetes.rst | 2 +- ...data-sharing-in-kubeflow-with-vineyard-csi-driver.rst} | 8 ++++---- go.work | 2 +- k8s/.gitignore | 2 +- k8s/Makefile | 6 ++---- k8s/cmd/README.md | 4 ---- k8s/test/csidriver/Dockerfile | 2 +- k8s/test/csidriver/Makefile | 2 +- 8 files changed, 11 insertions(+), 17 deletions(-) rename docs/tutorials/kubernetes/{speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst => efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst} (97%) diff --git a/docs/tutorials/kubernetes.rst b/docs/tutorials/kubernetes.rst index a4d84ed9..a7146e3d 100644 --- a/docs/tutorials/kubernetes.rst +++ b/docs/tutorials/kubernetes.rst @@ -9,7 +9,7 @@ Vineyard on Kubernetes ./kubernetes/using-vineyard-operator.rst ./kubernetes/ml-pipeline-mars-pytorch.rst ./kubernetes/data-sharing-with-vineyard-on-kubernetes.rst - ./kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst + ./kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst Vineyard can be seamlessly deployed on Kubernetes, managed by the :ref:`vineyard-operator`, to enhance big-data workflows through its data-aware scheduling policy. This policy diff --git a/docs/tutorials/kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst similarity index 97% rename from docs/tutorials/kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst rename to docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst index 7ffb65d4..b8305455 100644 --- a/docs/tutorials/kubernetes/speed-up-pipelines-based-on-volumes-with-vineyard-csi-driver.rst +++ b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst @@ -1,5 +1,5 @@ -Speed up pipelines based on volumes with Vineyard CSI Driver -============================================================ +Efficient data sharing in Kubeflow with Vineyard CSI Driver +=========================================================== If you are using `Kubeflow Pipeline`_ or `Argo Workflow`_ to manage your machine learning workflow, you may find that the data saving/loading to the volumes is slow. @@ -11,7 +11,7 @@ Prerequisites ------------- - A kubernetes cluster with version >= 1.25.10. If you don't have one by hand, you can refer to the -guide `Prepare the Kubernetes Cluster`_ to create one. +guide `Initialize Kubernetes Cluster`_ to create one. - Install the `Vineyardctl`_ by following the official guide. - Install the `Argo Workflow CLI`_ by following the official guide. @@ -306,6 +306,6 @@ Delete the data volume: .. _Kubeflow Pipeline: https://github.com/kubeflow/kubeflow .. _Argo Workflow: https://github.com/argoproj/argo-workflows -.. _Prepare the Kubernetes Cluster: https://v6d.io/tutorials/data-processing/accelerate-data-sharing-in-kedro.html#prepare-the-kubernetes-cluster +.. _Initialize Kubernetes Cluster: https://v6d.io/tutorials/kubernetes/using-vineyard-operator.html#step-0-optional-initialize-kubernetes-cluster .. _Vineyardctl: https://v6d.io/notes/developers/build-from-source.html#install-vineyardctl .. _Argo Workflow CLI: https://github.com/argoproj/argo-workflows/releases/ \ No newline at end of file diff --git a/go.work b/go.work index cec3d134..6931331d 100644 --- a/go.work +++ b/go.work @@ -2,4 +2,4 @@ go 1.19 use ./go/vineyard -use ./k8s \ No newline at end of file +use ./k8s diff --git a/k8s/.gitignore b/k8s/.gitignore index 20e4a3d2..73eb9826 100644 --- a/k8s/.gitignore +++ b/k8s/.gitignore @@ -25,4 +25,4 @@ vineyardctl *~ # vendor, used for code-generate only -/vendor/ \ No newline at end of file +/vendor/ diff --git a/k8s/Makefile b/k8s/Makefile index 3c71839c..ab232e5e 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -224,11 +224,9 @@ vendor: docker-build: cd .. && \ if docker build --help | grep -q load; then \ - docker build --load -f k8s/Dockerfile . \ - -t $(IMG); \ + docker build --load -f k8s/Dockerfile . -t $(IMG); \ else \ - docker build -f k8s/Dockerfile . \ - -t $(IMG); \ + docker build -f k8s/Dockerfile . -t $(IMG); \ fi docker-build-push-multi-arch: diff --git a/k8s/cmd/README.md b/k8s/cmd/README.md index a8b81240..4fb33561 100644 --- a/k8s/cmd/README.md +++ b/k8s/cmd/README.md @@ -2269,7 +2269,3 @@ vineyardctl schedule workload [flags] --vineyardd-name string the namespace of vineyard cluster (default "vineyardd-sample") --vineyardd-namespace string the namespace of vineyard cluster (default "vineyard-system") ``` - -stem") -``` - diff --git a/k8s/test/csidriver/Dockerfile b/k8s/test/csidriver/Dockerfile index 3b343423..f7f6d8f4 100644 --- a/k8s/test/csidriver/Dockerfile +++ b/k8s/test/csidriver/Dockerfile @@ -22,4 +22,4 @@ WORKDIR /workspace/k8s/cmd/commands/csi USER root -CMD ["go", "test", "-run", "./..."] \ No newline at end of file +CMD ["go", "test", "-run", "./..."] diff --git a/k8s/test/csidriver/Makefile b/k8s/test/csidriver/Makefile index b11c26a3..bb59ebf3 100644 --- a/k8s/test/csidriver/Makefile +++ b/k8s/test/csidriver/Makefile @@ -29,4 +29,4 @@ csidriver-test: docker build -f k8s/test/csidriver/Dockerfile . \ -t $(VINEYARD_CSI_TEST_IMAGE); \ fi && \ - docker run --rm -it --privileged=true $(VINEYARD_CSI_TEST_IMAGE) \ No newline at end of file + docker run --rm -it --privileged=true $(VINEYARD_CSI_TEST_IMAGE)