From be3df552bcbc898f39832ca825dcdd992289a13d Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 18 Nov 2024 15:45:25 +0800 Subject: [PATCH 1/5] Update merlin and mlflow versions --- charts/merlin/README.md | 7 ++++--- charts/merlin/values.yaml | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/charts/merlin/README.md b/charts/merlin/README.md index ab5c39e0..6ba5daf7 100644 --- a/charts/merlin/README.md +++ b/charts/merlin/README.md @@ -230,6 +230,7 @@ The following table lists the configurable parameters of the Merlin chart and th | imageBuilder.builderConfig.KanikoAdditionalArgs[3] | string | `"--use-new-run"` | | | imageBuilder.builderConfig.KanikoAdditionalArgs[4] | string | `"--log-timestamp"` | | | imageBuilder.builderConfig.KanikoImage | string | `"gcr.io/kaniko-project/executor:v1.18.0"` | | +| imageBuilder.builderConfig.KanikoPushRegistryType | string | `"gcr"` | | | imageBuilder.builderConfig.MaximumRetry | int | `3` | | | imageBuilder.builderConfig.NodeSelectors | object | `{}` | | | imageBuilder.builderConfig.PredictionJobBaseImage.BuildContextSubPath | string | `"python"` | | @@ -313,8 +314,8 @@ The following table lists the configurable parameters of the Merlin chart and th | mlflow.host | string | `"0.0.0.0"` | | | mlflow.image.pullPolicy | string | `"Always"` | | | mlflow.image.registry | string | `"ghcr.io"` | | -| mlflow.image.repository | string | `"gojek/mlflow"` | | -| mlflow.image.tag | string | `"1.3.0"` | | +| mlflow.image.repository | string | `"caraml-dev/mlflow"` | | +| mlflow.image.tag | string | `"1.26.1"` | | | mlflow.ingress.class | string | `"nginx"` | | | mlflow.ingress.enabled | bool | `false` | | | mlflow.livenessProbe.initialDelaySeconds | int | `30` | | @@ -361,7 +362,7 @@ The following table lists the configurable parameters of the Merlin chart and th | mlp.keto.enabled | bool | `true` | | | mlp.keto.fullnameOverride | string | `"mlp-keto"` | | | rendered.overrides | object | `{}` | | -| rendered.releasedVersion | string | `"v0.38.0-rc1"` | | +| rendered.releasedVersion | string | `"v0.45.3"` | | | service.externalPort | int | `8080` | | | service.internalPort | int | `8080` | | | serviceAccount.annotations | object | `{}` | | diff --git a/charts/merlin/values.yaml b/charts/merlin/values.yaml index 701f287a..1396a008 100644 --- a/charts/merlin/values.yaml +++ b/charts/merlin/values.yaml @@ -5,7 +5,7 @@ global: # .config when generating templates rendered: # releasedVersion refers to the git release or tag - releasedVersion: v0.38.0-rc1 + releasedVersion: v0.45.3 overrides: {} # set deployment.image.tag to non-nil to overwrite # .rendered.releasedVersion @@ -282,6 +282,7 @@ imageBuilder: DockerRegistry: "dockerRegistry" BuildTimeout: "30m" KanikoImage: "gcr.io/kaniko-project/executor:v1.18.0" + KanikoPushRegistryType: gcr KanikoAdditionalArgs: - "--cache=true" - "--compressed-caching=false" @@ -477,8 +478,8 @@ mlflow: podLabels: {} image: registry: ghcr.io - repository: gojek/mlflow - tag: 1.3.0 + repository: caraml-dev/mlflow + tag: 1.26.1 pullPolicy: Always replicaCount: 1 rollingUpdate: From 622285d02fd66375794cda13bb9b65199ce33b15 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 18 Nov 2024 15:51:00 +0800 Subject: [PATCH 2/5] Update merlin chart version --- charts/merlin/Chart.yaml | 2 +- charts/merlin/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/merlin/Chart.yaml b/charts/merlin/Chart.yaml index 0cda67cc..59158755 100644 --- a/charts/merlin/Chart.yaml +++ b/charts/merlin/Chart.yaml @@ -33,4 +33,4 @@ maintainers: - email: caraml-dev@caraml.dev name: caraml-dev name: merlin -version: 0.13.18 +version: 0.13.19 diff --git a/charts/merlin/README.md b/charts/merlin/README.md index 6ba5daf7..9c7bc5ed 100644 --- a/charts/merlin/README.md +++ b/charts/merlin/README.md @@ -1,7 +1,7 @@ # merlin --- -![Version: 0.13.18](https://img.shields.io/badge/Version-0.13.18-informational?style=flat-square) +![Version: 0.13.19](https://img.shields.io/badge/Version-0.13.19-informational?style=flat-square) ![AppVersion: v0.42.0](https://img.shields.io/badge/AppVersion-v0.42.0-informational?style=flat-square) Kubernetes-friendly ML model management, deployment, and serving. From b8618168e3ec692e7751ddde18b1d991b02454b6 Mon Sep 17 00:00:00 2001 From: ewezy Date: Mon, 18 Nov 2024 15:56:08 +0800 Subject: [PATCH 3/5] Update merlin versions --- charts/merlin/README.md | 8 ++++---- charts/merlin/tests/merlin_config_secret_test.yaml | 4 ++-- charts/merlin/tests/merlin_deployment_test.yaml | 4 ++-- charts/merlin/values.yaml | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/charts/merlin/README.md b/charts/merlin/README.md index 9c7bc5ed..096749ed 100644 --- a/charts/merlin/README.md +++ b/charts/merlin/README.md @@ -214,9 +214,9 @@ The following table lists the configurable parameters of the Merlin chart and th | global.protocol | string | `"http"` | | | imageBuilder.builderConfig.ArtifactServiceType | string | `"nop"` | | | imageBuilder.builderConfig.BaseImage.BuildContextSubPath | string | `"python"` | | -| imageBuilder.builderConfig.BaseImage.BuildContextURI | string | `"git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1"` | | +| imageBuilder.builderConfig.BaseImage.BuildContextURI | string | `"git://github.com/caraml-dev/merlin.git#refs/tags/v0.45.3"` | | | imageBuilder.builderConfig.BaseImage.DockerfilePath | string | `"pyfunc-server/docker/Dockerfile"` | | -| imageBuilder.builderConfig.BaseImage.ImageName | string | `"ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.38.0-rc1"` | | +| imageBuilder.builderConfig.BaseImage.ImageName | string | `"ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.45.3"` | | | imageBuilder.builderConfig.BuildNamespace | string | `"mlp"` | | | imageBuilder.builderConfig.BuildTimeout | string | `"30m"` | | | imageBuilder.builderConfig.DefaultResources.Limits.CPU | string | `"1"` | | @@ -234,9 +234,9 @@ The following table lists the configurable parameters of the Merlin chart and th | imageBuilder.builderConfig.MaximumRetry | int | `3` | | | imageBuilder.builderConfig.NodeSelectors | object | `{}` | | | imageBuilder.builderConfig.PredictionJobBaseImage.BuildContextSubPath | string | `"python"` | | -| imageBuilder.builderConfig.PredictionJobBaseImage.BuildContextURI | string | `"git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1"` | | +| imageBuilder.builderConfig.PredictionJobBaseImage.BuildContextURI | string | `"git://github.com/caraml-dev/merlin.git#refs/tags/v0.45.3"` | | | imageBuilder.builderConfig.PredictionJobBaseImage.DockerfilePath | string | `"batch-predictor/docker/app.Dockerfile"` | | -| imageBuilder.builderConfig.PredictionJobBaseImage.ImageName | string | `"ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.38.0-rc1"` | | +| imageBuilder.builderConfig.PredictionJobBaseImage.ImageName | string | `"ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.45.3"` | | | imageBuilder.builderConfig.PredictionJobBaseImage.MainAppPath | string | `"/home/spark/merlin-spark-app/main.py"` | | | imageBuilder.builderConfig.Retention | string | `"48h"` | | | imageBuilder.builderConfig.SafeToEvict | bool | `false` | | diff --git a/charts/merlin/tests/merlin_config_secret_test.yaml b/charts/merlin/tests/merlin_config_secret_test.yaml index c23f079d..af9c8fb8 100644 --- a/charts/merlin/tests/merlin_config_secret_test.yaml +++ b/charts/merlin/tests/merlin_config_secret_test.yaml @@ -195,7 +195,7 @@ tests: pattern: my-release-merlin-config$ - matchRegex: path: stringData.[config.yaml] - pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.38.0-rc1" + pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.45.3" - matchRegex: path: stringData.[config.yaml] - pattern: "BuildContextURI: git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1" + pattern: "BuildContextURI: git://github.com/caraml-dev/merlin.git#refs/tags/v0.45.3" diff --git a/charts/merlin/tests/merlin_deployment_test.yaml b/charts/merlin/tests/merlin_deployment_test.yaml index 4b283d58..6676beb7 100644 --- a/charts/merlin/tests/merlin_deployment_test.yaml +++ b/charts/merlin/tests/merlin_deployment_test.yaml @@ -177,7 +177,7 @@ tests: pattern: my-release-merlin$ - equal: # check image version path: spec.template.spec.containers[0].image - value: ghcr.io/caraml-dev/merlin:0.38.0-rc1 + value: ghcr.io/caraml-dev/merlin:0.45.3 - equal: # check version label path: spec.template.metadata.labels.version - value: 0.38.0-rc1 + value: 0.45.3 diff --git a/charts/merlin/values.yaml b/charts/merlin/values.yaml index 1396a008..a0715333 100644 --- a/charts/merlin/values.yaml +++ b/charts/merlin/values.yaml @@ -268,14 +268,14 @@ imageBuilder: builderConfig: ArtifactServiceType: nop BaseImage: - ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.38.0-rc1 + ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.45.3 DockerfilePath: "pyfunc-server/docker/Dockerfile" - BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1" + BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.45.3" BuildContextSubPath: "python" PredictionJobBaseImage: - ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.38.0-rc1 + ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.45.3 DockerfilePath: "batch-predictor/docker/app.Dockerfile" - BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1" + BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.45.3" BuildContextSubPath: "python" MainAppPath: "/home/spark/merlin-spark-app/main.py" BuildNamespace: "mlp" From 1ef5c73a3eae06fcd728855a247e862f79688ae8 Mon Sep 17 00:00:00 2001 From: ewezy Date: Tue, 19 Nov 2024 09:57:22 +0800 Subject: [PATCH 4/5] Add required observability publisher configs --- charts/merlin/README.md | 5 +++++ charts/merlin/values.yaml | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/charts/merlin/README.md b/charts/merlin/README.md index 096749ed..d6199e45 100644 --- a/charts/merlin/README.md +++ b/charts/merlin/README.md @@ -107,6 +107,11 @@ The following table lists the configurable parameters of the Merlin chart and th | config.NewRelic.IgnoreStatusCodes[5] | int | `412` | | | config.NewRelic.License | string | `"newrelic-license-secret"` | | | config.NumOfQueueWorkers | int | `2` | | +| config.ObservabilityPublisher.DefaultResources.Limits.CPU | string | `"2"` | | +| config.ObservabilityPublisher.DefaultResources.Limits.Memory | string | `"1Gi"` | | +| config.ObservabilityPublisher.DefaultResources.Requests.CPU | string | `"1"` | | +| config.ObservabilityPublisher.DefaultResources.Requests.Memory | string | `"1Gi"` | | +| config.ObservabilityPublisher.KafkaConsumer.Brokers | string | `"kafka-brokers"` | | | config.Port | int | `8080` | | | config.PyFuncPublisherConfig.Kafka.Acks | int | `0` | | | config.PyFuncPublisherConfig.Kafka.AdditionalConfig | string | `"{}"` | | diff --git a/charts/merlin/values.yaml b/charts/merlin/values.yaml index a0715333..9830a261 100644 --- a/charts/merlin/values.yaml +++ b/charts/merlin/values.yaml @@ -236,6 +236,16 @@ config: Acks: 0 LingerMS: 100 AdditionalConfig: "{}" + ObservabilityPublisher: + KafkaConsumer: + Brokers: "kafka-brokers" + DefaultResources: + Requests: + CPU: "1" + Memory: "1Gi" + Limits: + CPU: "2" + Memory: "1Gi" clusterConfig: # -- (bool) Configuration to tell Merlin API how it should authenticate with deployment k8s cluster # By default, Merlin API expects to use a remote k8s cluster for deployment and to do so, it requires From acf16272f800d8f63c1dc22dda3e4c635aa77380 Mon Sep 17 00:00:00 2001 From: ewezy Date: Tue, 19 Nov 2024 10:44:45 +0800 Subject: [PATCH 5/5] Add required environment name for observability publisher configs --- charts/merlin/README.md | 1 + charts/merlin/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/charts/merlin/README.md b/charts/merlin/README.md index d6199e45..29297dd4 100644 --- a/charts/merlin/README.md +++ b/charts/merlin/README.md @@ -111,6 +111,7 @@ The following table lists the configurable parameters of the Merlin chart and th | config.ObservabilityPublisher.DefaultResources.Limits.Memory | string | `"1Gi"` | | | config.ObservabilityPublisher.DefaultResources.Requests.CPU | string | `"1"` | | | config.ObservabilityPublisher.DefaultResources.Requests.Memory | string | `"1Gi"` | | +| config.ObservabilityPublisher.EnvironmentName | string | `"id-dev"` | | | config.ObservabilityPublisher.KafkaConsumer.Brokers | string | `"kafka-brokers"` | | | config.Port | int | `8080` | | | config.PyFuncPublisherConfig.Kafka.Acks | int | `0` | | diff --git a/charts/merlin/values.yaml b/charts/merlin/values.yaml index 9830a261..b609f955 100644 --- a/charts/merlin/values.yaml +++ b/charts/merlin/values.yaml @@ -246,6 +246,7 @@ config: Limits: CPU: "2" Memory: "1Gi" + EnvironmentName: id-dev clusterConfig: # -- (bool) Configuration to tell Merlin API how it should authenticate with deployment k8s cluster # By default, Merlin API expects to use a remote k8s cluster for deployment and to do so, it requires