feat: Object store persistence in operator (feast-dev#4758)

* object store validation and additional field Signed-off-by: Daniele Martinoli <[email protected]> * s3 additional kwargs Signed-off-by: Daniele Martinoli <[email protected]> * adding extra line Signed-off-by: Daniele Martinoli <[email protected]> * updated registry validation message Signed-off-by: Daniele Martinoli <[email protected]> * renamed S3AddtlKwargs to S3AdditionalKwargs Signed-off-by: Daniele Martinoli <[email protected]> * generalized similar test functions Signed-off-by: Daniele Martinoli <[email protected]> --------- Signed-off-by: Daniele Martinoli <[email protected]>
tmihalac · Nov 15, 2024 · 0ae86da · 0ae86da
1 parent d8ccb00
commit 0ae86da
Show file tree

Hide file tree

Showing 9 changed files with 634 additions and 18 deletions.
diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go
@@ -105,6 +105,7 @@ type OnlineStorePersistence struct {
 // OnlineStoreFilePersistence configures the file-based persistence for the offline store service
 // +kubebuilder:validation:XValidation:rule="(!has(self.pvc) && has(self.path)) ? self.path.startsWith('/') : true",message="Ephemeral stores must have absolute paths."
 // +kubebuilder:validation:XValidation:rule="(has(self.pvc) && has(self.path)) ? !self.path.startsWith('/') : true",message="PVC path must be a file name only, with no slashes."
+// +kubebuilder:validation:XValidation:rule="has(self.path) && !self.path.startsWith('s3://') && !self.path.startsWith('gs://')",message="Online store does not support S3 or GS buckets."
 type OnlineStoreFilePersistence struct {
 	Path      string     `json:"path,omitempty"`
 	PvcConfig *PvcConfig `json:"pvc,omitempty"`
@@ -122,11 +123,14 @@ type RegistryPersistence struct {
 }
 
 // RegistryFilePersistence configures the file-based persistence for the registry service
-// +kubebuilder:validation:XValidation:rule="(!has(self.pvc) && has(self.path)) ? self.path.startsWith('/') : true",message="Ephemeral stores must have absolute paths."
+// +kubebuilder:validation:XValidation:rule="(!has(self.pvc) && has(self.path)) ? (self.path.startsWith('/') || self.path.startsWith('s3://') || self.path.startsWith('gs://')) : true",message="Registry files must use absolute paths or be S3 ('s3://') or GS ('gs://') object store URIs."
 // +kubebuilder:validation:XValidation:rule="(has(self.pvc) && has(self.path)) ? !self.path.startsWith('/') : true",message="PVC path must be a file name only, with no slashes."
+// +kubebuilder:validation:XValidation:rule="(has(self.pvc) && has(self.path)) ? !(self.path.startsWith('s3://') || self.path.startsWith('gs://')) : true",message="PVC persistence does not support S3 or GS object store URIs."
+// +kubebuilder:validation:XValidation:rule="(has(self.s3_additional_kwargs) && has(self.path)) ? self.path.startsWith('s3://') : true",message="Additional S3 settings are available only for S3 object store URIs."
 type RegistryFilePersistence struct {
-	Path      string     `json:"path,omitempty"`
-	PvcConfig *PvcConfig `json:"pvc,omitempty"`
+	Path               string             `json:"path,omitempty"`
+	PvcConfig          *PvcConfig         `json:"pvc,omitempty"`
+	S3AdditionalKwargs *map[string]string `json:"s3_additional_kwargs,omitempty"`
 }
 
 // PvcConfig defines the settings for a persistent file store based on PVCs.

diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml
@@ -545,6 +545,9 @@ spec:
                                 slashes.
                               rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                 : true'
+                            - message: Online store does not support S3 or GS buckets.
+                              rule: has(self.path) && !self.path.startsWith('s3://')
+                                && !self.path.startsWith('gs://')
                         type: object
                       resources:
                         description: ResourceRequirements describes the compute resource
@@ -816,15 +819,30 @@ spec:
                                     - message: Mount path must start with '/' and
                                         must not contain ':'
                                       rule: self.mountPath.matches('^/[^:]*$')
+                                  s3_additional_kwargs:
+                                    additionalProperties:
+                                      type: string
+                                    type: object
                                 type: object
                                 x-kubernetes-validations:
-                                - message: Ephemeral stores must have absolute paths.
-                                  rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'')
+                                - message: Registry files must use absolute paths
+                                    or be S3 ('s3://') or GS ('gs://') object store
+                                    URIs.
+                                  rule: '(!has(self.pvc) && has(self.path)) ? (self.path.startsWith(''/'')
+                                    || self.path.startsWith(''s3://'') || self.path.startsWith(''gs://''))
                                     : true'
                                 - message: PVC path must be a file name only, with
                                     no slashes.
                                   rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                     : true'
+                                - message: PVC persistence does not support S3 or
+                                    GS object store URIs.
+                                  rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'')
+                                    || self.path.startsWith(''gs://'')) : true'
+                                - message: Additional S3 settings are available only
+                                    for S3 object store URIs.
+                                  rule: '(has(self.s3_additional_kwargs) && has(self.path))
+                                    ? self.path.startsWith(''s3://'') : true'
                             type: object
                           resources:
                             description: ResourceRequirements describes the compute
@@ -1427,6 +1445,10 @@ spec:
                                     no slashes.
                                   rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                     : true'
+                                - message: Online store does not support S3 or GS
+                                    buckets.
+                                  rule: has(self.path) && !self.path.startsWith('s3://')
+                                    && !self.path.startsWith('gs://')
                             type: object
                           resources:
                             description: ResourceRequirements describes the compute
@@ -1705,16 +1727,30 @@ spec:
                                         - message: Mount path must start with '/'
                                             and must not contain ':'
                                           rule: self.mountPath.matches('^/[^:]*$')
+                                      s3_additional_kwargs:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
                                     type: object
                                     x-kubernetes-validations:
-                                    - message: Ephemeral stores must have absolute
-                                        paths.
+                                    - message: Registry files must use absolute paths
+                                        or be S3 ('s3://') or GS ('gs://') object
+                                        store URIs.
                                       rule: '(!has(self.pvc) && has(self.path)) ?
-                                        self.path.startsWith(''/'') : true'
+                                        (self.path.startsWith(''/'') || self.path.startsWith(''s3://'')
+                                        || self.path.startsWith(''gs://'')) : true'
                                     - message: PVC path must be a file name only,
                                         with no slashes.
                                       rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                         : true'
+                                    - message: PVC persistence does not support S3
+                                        or GS object store URIs.
+                                      rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'')
+                                        || self.path.startsWith(''gs://'')) : true'
+                                    - message: Additional S3 settings are available
+                                        only for S3 object store URIs.
+                                      rule: '(has(self.s3_additional_kwargs) && has(self.path))
+                                        ? self.path.startsWith(''s3://'') : true'
                                 type: object
                               resources:
                                 description: ResourceRequirements describes the compute

diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_objectstore_persistence.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_objectstore_persistence.yaml
@@ -0,0 +1,24 @@
+apiVersion: feast.dev/v1alpha1
+kind: FeatureStore
+metadata:
+  name: sample-s3-registry
+spec:
+  feastProject: my_project
+  services:
+    onlineStore:
+      persistence:
+        file:
+          path: /data/online_store.db
+    offlineStore:
+      persistence:
+        file:
+          type: dask
+    registry:
+      local:
+        persistence:
+          file:
+            path: s3://bucket/registry.db
+            s3_additional_kwargs:
+              ServerSideEncryption: AES256
+              ACL: bucket-owner-full-control
+              CacheControl: max-age=3600
diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml
@@ -553,6 +553,9 @@ spec:
                                 slashes.
                               rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                 : true'
+                            - message: Online store does not support S3 or GS buckets.
+                              rule: has(self.path) && !self.path.startsWith('s3://')
+                                && !self.path.startsWith('gs://')
                         type: object
                       resources:
                         description: ResourceRequirements describes the compute resource
@@ -824,15 +827,30 @@ spec:
                                     - message: Mount path must start with '/' and
                                         must not contain ':'
                                       rule: self.mountPath.matches('^/[^:]*$')
+                                  s3_additional_kwargs:
+                                    additionalProperties:
+                                      type: string
+                                    type: object
                                 type: object
                                 x-kubernetes-validations:
-                                - message: Ephemeral stores must have absolute paths.
-                                  rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'')
+                                - message: Registry files must use absolute paths
+                                    or be S3 ('s3://') or GS ('gs://') object store
+                                    URIs.
+                                  rule: '(!has(self.pvc) && has(self.path)) ? (self.path.startsWith(''/'')
+                                    || self.path.startsWith(''s3://'') || self.path.startsWith(''gs://''))
                                     : true'
                                 - message: PVC path must be a file name only, with
                                     no slashes.
                                   rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                     : true'
+                                - message: PVC persistence does not support S3 or
+                                    GS object store URIs.
+                                  rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'')
+                                    || self.path.startsWith(''gs://'')) : true'
+                                - message: Additional S3 settings are available only
+                                    for S3 object store URIs.
+                                  rule: '(has(self.s3_additional_kwargs) && has(self.path))
+                                    ? self.path.startsWith(''s3://'') : true'
                             type: object
                           resources:
                             description: ResourceRequirements describes the compute
@@ -1435,6 +1453,10 @@ spec:
                                     no slashes.
                                   rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                     : true'
+                                - message: Online store does not support S3 or GS
+                                    buckets.
+                                  rule: has(self.path) && !self.path.startsWith('s3://')
+                                    && !self.path.startsWith('gs://')
                             type: object
                           resources:
                             description: ResourceRequirements describes the compute
@@ -1713,16 +1735,30 @@ spec:
                                         - message: Mount path must start with '/'
                                             and must not contain ':'
                                           rule: self.mountPath.matches('^/[^:]*$')
+                                      s3_additional_kwargs:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
                                     type: object
                                     x-kubernetes-validations:
-                                    - message: Ephemeral stores must have absolute
-                                        paths.
+                                    - message: Registry files must use absolute paths
+                                        or be S3 ('s3://') or GS ('gs://') object
+                                        store URIs.
                                       rule: '(!has(self.pvc) && has(self.path)) ?
-                                        self.path.startsWith(''/'') : true'
+                                        (self.path.startsWith(''/'') || self.path.startsWith(''s3://'')
+                                        || self.path.startsWith(''gs://'')) : true'
                                     - message: PVC path must be a file name only,
                                         with no slashes.
                                       rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'')
                                         : true'
+                                    - message: PVC persistence does not support S3
+                                        or GS object store URIs.
+                                      rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'')
+                                        || self.path.startsWith(''gs://'')) : true'
+                                    - message: Additional S3 settings are available
+                                        only for S3 object store URIs.
+                                      rule: '(has(self.s3_additional_kwargs) && has(self.path))
+                                        ? self.path.startsWith(''s3://'') : true'
                                 type: object
                               resources:
                                 description: ResourceRequirements describes the compute