feat: Add job for ML observability workflow registration

charts/merlin/Chart.yaml

@@ -33,4 +33,4 @@ maintainers:
 - email: [email protected]
   name: caraml-dev
 name: merlin
-version: 0.13.12
+version: 0.14.0

charts/merlin/README.md

@@ -1,7 +1,7 @@
 # merlin
 
 ---
-![Version: 0.13.12](https://img.shields.io/badge/Version-0.13.12-informational?style=flat-square)
+![Version: 0.14.0](https://img.shields.io/badge/Version-0.14.0-informational?style=flat-square)
 ![AppVersion: v0.38.0-rc1](https://img.shields.io/badge/AppVersion-v0.38.0--rc1-informational?style=flat-square)
 
 Kubernetes-friendly ML model management, deployment, and serving.
@@ -351,6 +351,14 @@ The following table lists the configurable parameters of the Merlin chart and th
 | mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy | object | `{"dbConnectionName":"asia-east-1:mlflow-db","dbPort":5432,"image":{"tag":"1.33.2"},"resources":{"limits":{"cpu":"1000m","memory":"1G"},"requests":{"cpu":"200m","memory":"512Mi"}},"spec":[{"command":["/cloud_sql_proxy","-ip_address_types=PRIVATE","-log_debug_stdout","-instances={{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbConnectionName }}=tcp:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbPort }}"],"image":"gcr.io/cloudsql-docker/gce-proxy:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.image.tag }}","name":"cloud-sql-proxy","resources":{"limits":{"cpu":"1000m","memory":"1G"},"requests":{"cpu":"200m","memory":"512Mi"}},"securityContext":{"runAsNonRoot":true}}]}` | container spec for the Google CloudSQL auth proxy sidecar, ref: https://cloud.google.com/sql/docs/postgres/connect-kubernetes-engine |
 | mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.spec | list | `[{"command":["/cloud_sql_proxy","-ip_address_types=PRIVATE","-log_debug_stdout","-instances={{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbConnectionName }}=tcp:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbPort }}"],"image":"gcr.io/cloudsql-docker/gce-proxy:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.image.tag }}","name":"cloud-sql-proxy","resources":{"limits":{"cpu":"1000m","memory":"1G"},"requests":{"cpu":"200m","memory":"512Mi"}},"securityContext":{"runAsNonRoot":true}}]` | Container spec for the sidecar |
 | mlflowExternalPostgresql.username | string | `"mlflow"` | External postgres database user |
+| mlobs.bootstrap.enabled | bool | `false` | Enable this to run bootstrap job for mlobs workflow registration |
+| mlobs.bootstrap.flyte.config | object | `{"admin":{"endpoint":"flyte.example.com:80","insecure":true}}` | Flyte cluster configuration, which we will register the workflow on |
+| mlobs.bootstrap.flyte.domain | string | `"development"` | Flyte workflow domain |
+| mlobs.bootstrap.flyte.project | string | `"merlin"` | Flyte workflow project |
+| mlobs.bootstrap.flyte.workflowImageName | string | `"caraml-dev/merlin/merlin-batch-observation-publisher"` | Image to be used for both the bootstrap job pod and the Flyte workflow. |
+| mlobs.bootstrap.flyte.workflowImageTagOverride | string | `""` | Flyte workflow image tag override. If not defined, the deployment tag will be used. |
+| mlobs.bootstrap.resources.requests.cpu | string | `"100m"` |  |
+| mlobs.bootstrap.resources.requests.memory | string | `"100Mi"` |  |
 | mlp.enabled | bool | `true` |  |
 | mlp.environmentConfigSecret.name | string | `""` |  |
 | mlp.fullnameOverride | string | `"mlp"` |  |

charts/merlin/templates/mlobs-bootstrap-job-configmap.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.mlobs.bootstrap.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "merlin.resource-prefix" . }}-mlobs-bootstrap
+data:
+  config.yaml: |
+    {{- with .Values.mlobs.bootstrap.flyte.config }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- end -}}

charts/merlin/templates/mlobs-bootstrap-job.yaml

@@ -0,0 +1,38 @@
+{{- if .Values.mlobs.bootstrap.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "merlin.resource-prefix" . }}-mlobs-bootstrap
+  annotations:
+    "helm.sh/hook": post-upgrade,post-install
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  template:
+    spec:
+      containers:
+      - name: mlobs-bootstrap
+        image: "{{ .Values.deployment.image.registry }}/{{ .Values.mlobs.bootstrap.flyte.workflowImageName }}:{{ .Values.mlobs.bootstrap.flyte.workflowImageTagOverride | default .Values.deployment.image.tag }}"
+        command:
+          - make
+          - register
+        env:
+        - name: FLYTE_PROJECT
+          value: {{ .Values.mlobs.bootstrap.flyte.project }}
+        - name: FLYTE_DOMAIN
+          value: {{ .Values.mlobs.bootstrap.flyte.domain }}
+        - name: WORKFLOW_IMAGE
+          value: "{{ .Values.deployment.image.registry }}/{{ .Values.mlobs.bootstrap.flyte.workflowImageName }}:{{ .Values.mlobs.bootstrap.flyte.workflowImageTagOverride | default .Values.deployment.image.tag }}"
+        - name: WORKFLOW_VERSION
+          value: {{ .Values.mlobs.bootstrap.flyte.workflowImageTagOverride | default .Values.deployment.image.tag }}
+        resources:
+{{- toYaml .Values.mlobs.bootstrap.resources | nindent 10 }}
+        volumeMounts:
+        - name: flyte-config
+          mountPath: /root/.flyte/
+          readOnly: true
+      restartPolicy: Never
+      volumes:
+      - name: flyte-config
+        configMap:
+          name:  {{ template "merlin.resource-prefix" . }}-mlobs-bootstrap
+{{- end -}}

charts/merlin/values.yaml

@@ -691,3 +691,25 @@ kserve:
         istioIngressGateway:
           helmChart:
             namespace: "istio-system"
+mlobs:
+  bootstrap:
+    # -- Enable this to run bootstrap job for mlobs workflow registration
+    enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 100Mi
+    flyte:
+      # -- Flyte workflow project
+      project: merlin
+      # -- Flyte workflow domain
+      domain: development
+      # -- Image to be used for both the bootstrap job pod and the Flyte workflow.
+      workflowImageName: caraml-dev/merlin/merlin-batch-observation-publisher
+      # -- Flyte workflow image tag override. If not defined, the deployment tag will be used.
+      workflowImageTagOverride: ""
+      # -- Flyte cluster configuration, which we will register the workflow on
+      config:
+        admin:
+          endpoint: flyte.example.com:80
+          insecure: true