From 611513b5d7484be2781467f0e1bac68b4da2073a Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Fri, 5 Jan 2024 14:28:37 +0800 Subject: [PATCH] feat: Add job for ML observability workflow registration --- charts/merlin/Chart.yaml | 2 +- charts/merlin/README.md | 10 ++++- .../mlobs-bootstrap-job-configmap.yaml | 11 ++++++ .../merlin/templates/mlobs-bootstrap-job.yaml | 38 +++++++++++++++++++ charts/merlin/values.yaml | 22 +++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 charts/merlin/templates/mlobs-bootstrap-job-configmap.yaml create mode 100644 charts/merlin/templates/mlobs-bootstrap-job.yaml diff --git a/charts/merlin/Chart.yaml b/charts/merlin/Chart.yaml index 3fe5d1a5..34fdcbdc 100644 --- a/charts/merlin/Chart.yaml +++ b/charts/merlin/Chart.yaml @@ -33,4 +33,4 @@ maintainers: - email: caraml-dev@caraml.dev name: caraml-dev name: merlin -version: 0.13.12 +version: 0.14.0 diff --git a/charts/merlin/README.md b/charts/merlin/README.md index d5ad2813..af5710f2 100644 --- a/charts/merlin/README.md +++ b/charts/merlin/README.md @@ -1,7 +1,7 @@ # merlin --- -![Version: 0.13.12](https://img.shields.io/badge/Version-0.13.12-informational?style=flat-square) +![Version: 0.14.0](https://img.shields.io/badge/Version-0.14.0-informational?style=flat-square) ![AppVersion: v0.38.0-rc1](https://img.shields.io/badge/AppVersion-v0.38.0--rc1-informational?style=flat-square) Kubernetes-friendly ML model management, deployment, and serving. @@ -351,6 +351,14 @@ The following table lists the configurable parameters of the Merlin chart and th | mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy | object | `{"dbConnectionName":"asia-east-1:mlflow-db","dbPort":5432,"image":{"tag":"1.33.2"},"resources":{"limits":{"cpu":"1000m","memory":"1G"},"requests":{"cpu":"200m","memory":"512Mi"}},"spec":[{"command":["/cloud_sql_proxy","-ip_address_types=PRIVATE","-log_debug_stdout","-instances={{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbConnectionName }}=tcp:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbPort }}"],"image":"gcr.io/cloudsql-docker/gce-proxy:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.image.tag }}","name":"cloud-sql-proxy","resources":{"limits":{"cpu":"1000m","memory":"1G"},"requests":{"cpu":"200m","memory":"512Mi"}},"securityContext":{"runAsNonRoot":true}}]}` | container spec for the Google CloudSQL auth proxy sidecar, ref: https://cloud.google.com/sql/docs/postgres/connect-kubernetes-engine | | mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.spec | list | `[{"command":["/cloud_sql_proxy","-ip_address_types=PRIVATE","-log_debug_stdout","-instances={{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbConnectionName }}=tcp:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.dbPort }}"],"image":"gcr.io/cloudsql-docker/gce-proxy:{{ .Values.mlflowExternalPostgresql.sidecarSpec.cloudSqlProxy.image.tag }}","name":"cloud-sql-proxy","resources":{"limits":{"cpu":"1000m","memory":"1G"},"requests":{"cpu":"200m","memory":"512Mi"}},"securityContext":{"runAsNonRoot":true}}]` | Container spec for the sidecar | | mlflowExternalPostgresql.username | string | `"mlflow"` | External postgres database user | +| mlobs.bootstrap.enabled | bool | `false` | Enable this to run bootstrap job for mlobs workflow registration | +| mlobs.bootstrap.flyte.config | object | `{"admin":{"endpoint":"flyte.example.com:80","insecure":true}}` | Flyte cluster configuration, which we will register the workflow on | +| mlobs.bootstrap.flyte.domain | string | `"development"` | Flyte workflow domain | +| mlobs.bootstrap.flyte.project | string | `"merlin"` | Flyte workflow project | +| mlobs.bootstrap.flyte.workflowImageName | string | `"caraml-dev/merlin/merlin-batch-observation-publisher"` | Image to be used for both the bootstrap job pod and the Flyte workflow. | +| mlobs.bootstrap.flyte.workflowImageTagOverride | string | `""` | Flyte workflow image tag override. If not defined, the deployment tag will be used. | +| mlobs.bootstrap.resources.requests.cpu | string | `"100m"` | | +| mlobs.bootstrap.resources.requests.memory | string | `"100Mi"` | | | mlp.enabled | bool | `true` | | | mlp.environmentConfigSecret.name | string | `""` | | | mlp.fullnameOverride | string | `"mlp"` | | diff --git a/charts/merlin/templates/mlobs-bootstrap-job-configmap.yaml b/charts/merlin/templates/mlobs-bootstrap-job-configmap.yaml new file mode 100644 index 00000000..c4548916 --- /dev/null +++ b/charts/merlin/templates/mlobs-bootstrap-job-configmap.yaml @@ -0,0 +1,11 @@ +{{- if .Values.mlobs.bootstrap.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "merlin.resource-prefix" . }}-mlobs-bootstrap +data: + config.yaml: | + {{- with .Values.mlobs.bootstrap.flyte.config }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/merlin/templates/mlobs-bootstrap-job.yaml b/charts/merlin/templates/mlobs-bootstrap-job.yaml new file mode 100644 index 00000000..a5fd7319 --- /dev/null +++ b/charts/merlin/templates/mlobs-bootstrap-job.yaml @@ -0,0 +1,38 @@ +{{- if .Values.mlobs.bootstrap.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "merlin.resource-prefix" . }}-mlobs-bootstrap + annotations: + "helm.sh/hook": post-upgrade,post-install + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + spec: + containers: + - name: mlobs-bootstrap + image: "{{ .Values.deployment.image.registry }}/{{ .Values.mlobs.bootstrap.flyte.workflowImageName }}:{{ .Values.mlobs.bootstrap.flyte.workflowImageTagOverride | default .Values.deployment.image.tag }}" + command: + - make + - register + env: + - name: FLYTE_PROJECT + value: {{ .Values.mlobs.bootstrap.flyte.project }} + - name: FLYTE_DOMAIN + value: {{ .Values.mlobs.bootstrap.flyte.domain }} + - name: WORKFLOW_IMAGE + value: "{{ .Values.deployment.image.registry }}/{{ .Values.mlobs.bootstrap.flyte.workflowImageName }}:{{ .Values.mlobs.bootstrap.flyte.workflowImageTagOverride | default .Values.deployment.image.tag }}" + - name: WORKFLOW_VERSION + value: {{ .Values.mlobs.bootstrap.flyte.workflowImageTagOverride | default .Values.deployment.image.tag }} + resources: +{{- toYaml .Values.mlobs.bootstrap.resources | nindent 10 }} + volumeMounts: + - name: flyte-config + mountPath: /root/.flyte/ + readOnly: true + restartPolicy: Never + volumes: + - name: flyte-config + configMap: + name: {{ template "merlin.resource-prefix" . }}-mlobs-bootstrap +{{- end -}} diff --git a/charts/merlin/values.yaml b/charts/merlin/values.yaml index 798f074e..40c64178 100644 --- a/charts/merlin/values.yaml +++ b/charts/merlin/values.yaml @@ -691,3 +691,25 @@ kserve: istioIngressGateway: helmChart: namespace: "istio-system" +mlobs: + bootstrap: + # -- Enable this to run bootstrap job for mlobs workflow registration + enabled: false + resources: + requests: + cpu: 100m + memory: 100Mi + flyte: + # -- Flyte workflow project + project: merlin + # -- Flyte workflow domain + domain: development + # -- Image to be used for both the bootstrap job pod and the Flyte workflow. + workflowImageName: caraml-dev/merlin/merlin-batch-observation-publisher + # -- Flyte workflow image tag override. If not defined, the deployment tag will be used. + workflowImageTagOverride: "" + # -- Flyte cluster configuration, which we will register the workflow on + config: + admin: + endpoint: flyte.example.com:80 + insecure: true