chore(merlin): Update Merlin for Image Builder Improvements

charts/caraml/Chart.lock

@@ -45,4 +45,4 @@ dependencies:
   repository: https://charts.jetstack.io
   version: v1.8.2
 digest: sha256:63345d3cdfdb506d5ca52c9361aadaa58410f9ccdda78e263832dd0b4e875cdd
-generated: "2023-12-07T08:59:09.454202719Z"
+generated: "2023-12-07T08:59:09.454202719Z"

charts/caraml/Chart.yaml

@@ -72,4 +72,4 @@ maintainers:
   name: caraml-dev
 name: caraml
 type: application
-version: 0.8.20
+version: 0.8.21

charts/caraml/README.md

@@ -1,6 +1,6 @@
 # caraml
 
-![Version: 0.8.20](https://img.shields.io/badge/Version-0.8.20-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square)
+![Version: 0.8.21](https://img.shields.io/badge/Version-0.8.21-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square)
 
 A Helm chart for deploying CaraML components
 

charts/merlin/Chart.yaml

@@ -1,36 +1,36 @@
 apiVersion: v2
-appVersion: v0.36.0
+appVersion: v0.38.0-rc1
 dependencies:
-- alias: merlin-postgresql
-  condition: merlin-postgresql.enabled
-  name: postgresql
-  repository: https://charts.helm.sh/stable
-  version: 7.0.2
-- alias: mlflow-postgresql
-  condition: mlflow-postgresql.enabled
-  name: postgresql
-  repository: https://charts.helm.sh/stable
-  version: 7.0.2
-- condition: mlp.enabled
-  name: mlp
-  repository: https://caraml-dev.github.io/helm-charts
-  version: 0.6.4
-- alias: kserve
-  condition: kserve.enabled
-  name: generic-dep-installer
-  repository: https://caraml-dev.github.io/helm-charts
-  version: 0.2.1
-- alias: minio
-  condition: minio.enabled
-  name: generic-dep-installer
-  repository: https://caraml-dev.github.io/helm-charts
-  version: 0.2.1
-- name: common
-  repository: https://caraml-dev.github.io/helm-charts
-  version: 0.2.9
+  - alias: merlin-postgresql
+    condition: merlin-postgresql.enabled
+    name: postgresql
+    repository: https://charts.helm.sh/stable
+    version: 7.0.2
+  - alias: mlflow-postgresql
+    condition: mlflow-postgresql.enabled
+    name: postgresql
+    repository: https://charts.helm.sh/stable
+    version: 7.0.2
+  - condition: mlp.enabled
+    name: mlp
+    repository: https://caraml-dev.github.io/helm-charts
+    version: 0.6.4
+  - alias: kserve
+    condition: kserve.enabled
+    name: generic-dep-installer
+    repository: https://caraml-dev.github.io/helm-charts
+    version: 0.2.1
+  - alias: minio
+    condition: minio.enabled
+    name: generic-dep-installer
+    repository: https://caraml-dev.github.io/helm-charts
+    version: 0.2.1
+  - name: common
+    repository: https://caraml-dev.github.io/helm-charts
+    version: 0.2.9
 description: Kubernetes-friendly ML model management, deployment, and serving.
 maintainers:
-- email: [email protected]
-  name: caraml-dev
+  - email: [email protected]
+    name: caraml-dev
 name: merlin
-version: 0.13.7
+version: 0.13.8

charts/merlin/README.md

@@ -1,8 +1,8 @@
 # merlin
 
 ---
-![Version: 0.13.7](https://img.shields.io/badge/Version-0.13.7-informational?style=flat-square)
-![AppVersion: v0.36.0](https://img.shields.io/badge/AppVersion-v0.36.0-informational?style=flat-square)
+![Version: 0.13.8](https://img.shields.io/badge/Version-0.13.8-informational?style=flat-square)
+![AppVersion: v0.38.0-rc1](https://img.shields.io/badge/AppVersion-v0.38.0--rc1-informational?style=flat-square)
 
 Kubernetes-friendly ML model management, deployment, and serving.
 
@@ -208,24 +208,31 @@ The following table lists the configurable parameters of the Merlin chart and th
 | environmentConfigs[0].queue_resource_percentage | string | `"20"` |  |
 | environmentConfigs[0].region | string | `"id"` |  |
 | global.protocol | string | `"http"` |  |
-| imageBuilder.builderConfig.BaseImages."3.8.*".BuildContextURI | string | `"git://github.com/gojek/merlin.git#refs/tags/v0.1"` |  |
-| imageBuilder.builderConfig.BaseImages."3.8.*".DockerfilePath | string | `"docker/Dockerfile"` |  |
-| imageBuilder.builderConfig.BaseImages."3.8.*".ImageName | string | `"pyfunc-py38:v0.1.0"` |  |
-| imageBuilder.builderConfig.BaseImages."3.8.*".MainAppPath | string | `"/merlin-spark-app/main.py"` |  |
+| imageBuilder.builderConfig.ArtifactServiceType | string | `"nop"` |  |
+| imageBuilder.builderConfig.BaseImage.BuildContextSubPath | string | `"python"` |  |
+| imageBuilder.builderConfig.BaseImage.BuildContextURI | string | `"git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1"` |  |
+| imageBuilder.builderConfig.BaseImage.DockerfilePath | string | `"pyfunc-server/docker/Dockerfile"` |  |
+| imageBuilder.builderConfig.BaseImage.ImageName | string | `"ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.38.0-rc1"` |  |
 | imageBuilder.builderConfig.BuildNamespace | string | `"mlp"` |  |
 | imageBuilder.builderConfig.BuildTimeout | string | `"30m"` |  |
+| imageBuilder.builderConfig.DefaultResources.Limits.CPU | string | `"1"` |  |
+| imageBuilder.builderConfig.DefaultResources.Limits.Memory | string | `"1Gi"` |  |
+| imageBuilder.builderConfig.DefaultResources.Requests.CPU | string | `"1"` |  |
+| imageBuilder.builderConfig.DefaultResources.Requests.Memory | string | `"512Mi"` |  |
 | imageBuilder.builderConfig.DockerRegistry | string | `"dockerRegistry"` |  |
-| imageBuilder.builderConfig.KanikoImage | string | `"gcr.io/kaniko-project/executor:v1.6.0"` |  |
+| imageBuilder.builderConfig.KanikoAdditionalArgs[0] | string | `"--cache=true"` |  |
+| imageBuilder.builderConfig.KanikoAdditionalArgs[1] | string | `"--compressed-caching=false"` |  |
+| imageBuilder.builderConfig.KanikoAdditionalArgs[2] | string | `"--snapshot-mode=redo"` |  |
+| imageBuilder.builderConfig.KanikoAdditionalArgs[3] | string | `"--use-new-run"` |  |
+| imageBuilder.builderConfig.KanikoAdditionalArgs[4] | string | `"--log-timestamp"` |  |
+| imageBuilder.builderConfig.KanikoImage | string | `"gcr.io/kaniko-project/executor:v1.18.0"` |  |
 | imageBuilder.builderConfig.MaximumRetry | int | `3` |  |
 | imageBuilder.builderConfig.NodeSelectors | object | `{}` |  |
-| imageBuilder.builderConfig.PredictionJobBaseImages."3.8.*".BuildContextURI | string | `"git://github.com/gojek/merlin.git#refs/tags/v0.1"` |  |
-| imageBuilder.builderConfig.PredictionJobBaseImages."3.8.*".DockerfilePath | string | `"docker/app.Dockerfile"` |  |
-| imageBuilder.builderConfig.PredictionJobBaseImages."3.8.*".ImageName | string | `"pyspark-py38:v0.1.0"` |  |
-| imageBuilder.builderConfig.PredictionJobBaseImages."3.8.*".MainAppPath | string | `"/merlin-spark-app/main.py"` |  |
-| imageBuilder.builderConfig.Resources.Limits.CPU | string | `"1"` |  |
-| imageBuilder.builderConfig.Resources.Limits.Memory | string | `"1Gi"` |  |
-| imageBuilder.builderConfig.Resources.Requests.CPU | string | `"1"` |  |
-| imageBuilder.builderConfig.Resources.Requests.Memory | string | `"512Mi"` |  |
+| imageBuilder.builderConfig.PredictionJobBaseImages.BuildContextSubPath | string | `"python"` |  |
+| imageBuilder.builderConfig.PredictionJobBaseImages.BuildContextURI | string | `"git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1"` |  |
+| imageBuilder.builderConfig.PredictionJobBaseImages.DockerfilePath | string | `"batch-predictor/docker/app.Dockerfile"` |  |
+| imageBuilder.builderConfig.PredictionJobBaseImages.ImageName | string | `"ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.38.0-rc1"` |  |
+| imageBuilder.builderConfig.PredictionJobBaseImages.MainAppPath | string | `"/home/spark/merlin-spark-app/main.py"` |  |
 | imageBuilder.builderConfig.Retention | string | `"48h"` |  |
 | imageBuilder.builderConfig.SafeToEvict | bool | `false` |  |
 | imageBuilder.builderConfig.Tolerations | list | `[]` |  |
@@ -350,7 +357,7 @@ The following table lists the configurable parameters of the Merlin chart and th
 | mlp.keto.enabled | bool | `true` |  |
 | mlp.keto.fullnameOverride | string | `"mlp-keto"` |  |
 | rendered.overrides | object | `{}` |  |
-| rendered.releasedVersion | string | `"v0.35.0"` |  |
+| rendered.releasedVersion | string | `"v0.38.0-rc1"` |  |
 | service.externalPort | int | `8080` |  |
 | service.internalPort | int | `8080` |  |
 | serviceAccount.annotations | object | `{}` |  |

charts/merlin/templates/_config.tpl

@@ -16,35 +16,17 @@
 # Now we have access to the "real" root and current contexts
 # just as if we were outside of include/define:
 ImageBuilderConfig:
-  BaseImages:
-    3.8.*:
-      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py38:{{ printf "%s" $tag }}
-      DockerfilePath: "python/pyfunc-server/docker/Dockerfile"
-      BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
-    3.9.*:
-      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py39:{{ printf "%s" $tag }}
-      DockerfilePath: "python/pyfunc-server/docker/Dockerfile"
-      BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
-    3.10.*:
-      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py310:{{ printf "%s" $tag }}
-      DockerfilePath: "python/pyfunc-server/docker/Dockerfile"
-      BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
-  PredictionJobBaseImages:
-    3.8.*:
-      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py38:{{ printf "%s" $tag }}
-      DockerfilePath: "python/batch-predictor/docker/app.Dockerfile"
-      BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
-      MainAppPath: "/home/spark/merlin-spark-app/main.py"
-    3.9.*:
-      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py39:{{ printf "%s" $tag }}
-      DockerfilePath: "python/batch-predictor/docker/app.Dockerfile"
-      BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
-      MainAppPath: "/home/spark/merlin-spark-app/main.py"
-    3.10.*:
-      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py310:{{ printf "%s" $tag }}
-      DockerfilePath: "python/batch-predictor/docker/app.Dockerfile"
-      BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
-      MainAppPath: "/home/spark/merlin-spark-app/main.py"
+  BaseImage:
+    ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:{{ printf "%s" $tag }}
+    DockerfilePath: "pyfunc-server/docker/Dockerfile"
+    BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
+    BuildContextSubPath: "python"
+  PredictionJobBaseImage:
+    ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base:{{ printf "%s" $tag }}
+    DockerfilePath: "batch-predictor/docker/app.Dockerfile"
+    BuildContextURI: "git://github.com/caraml-dev/merlin.git#{{ printf "%s" $reference }}"
+    BuildContextSubPath: "python"
+    MainAppPath: "/home/spark/merlin-spark-app/main.py"
 StandardTransformerConfig:
   ImageName: ghcr.io/caraml-dev/merlin-transformer:{{ printf "%s" $tag }}
 {{- end -}}

charts/merlin/tests/merlin_config_secret_test.yaml

@@ -124,22 +124,10 @@ tests:
           pattern: my-release-merlin-config$
       - matchRegex:
           path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py38:1.0.0-test-release"
+          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:1.0.0-test-release"
       - matchRegex:
           path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py39:1.0.0-test-release"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py310:1.0.0-test-release"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py38:1.0.0-test-release"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py39:1.0.0-test-release"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py310:1.0.0-test-release"
+          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base:1.0.0-test-release"
       - matchRegex:
           path: stringData.[config.yaml]
           pattern: "ImageName: ghcr.io/caraml-dev/merlin-transformer:1.0.0-test-release"
@@ -164,11 +152,10 @@ tests:
         releasedVersion: v1.0.0-test-release
         overrides:
           ImageBuilderConfig:
-            BaseImages:
-              3.10.*:
-                ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py310:0.0.0-some-other-value
-                DockerfilePath: "python/pyfunc-server/docker/Dockerfile"
-                BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/pull/465"
+            BaseImage:
+              ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.0.0-some-other-value
+              DockerfilePath: "python/pyfunc-server/docker/Dockerfile"
+              BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/pull/465"
     asserts:
       - isKind:
           of: Secret
@@ -177,22 +164,10 @@ tests:
           pattern: my-release-merlin-config$
       - matchRegex:
           path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py38:1.0.0-test-release"
+          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.0.0-some-other-value"
       - matchRegex:
           path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py39:1.0.0-test-release"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py310:0.0.0-some-other-value"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py38:1.0.0-test-release"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py39:1.0.0-test-release"
-      - matchRegex:
-          path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base-py310:1.0.0-test-release"
+          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base:1.0.0-test-release"
       - matchRegex:
           path: stringData.[config.yaml]
           pattern: "ImageName: ghcr.io/caraml-dev/merlin-transformer:1.0.0-test-release"
@@ -208,11 +183,10 @@ tests:
           tag: "test-tag"
       config:
         ImageBuilderConfig:
-          BaseImages:
-            3.10.*:
-              ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py310:0.0.0-set-from-config
-              DockerfilePath: "python/pyfunc-server/docker/Dockerfile"
-              BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/pull/123"
+          BaseImage:
+            ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.0.0-set-from-config
+            DockerfilePath: "python/pyfunc-server/docker/Dockerfile"
+            BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/pull/123"
     asserts:
       - isKind:
           of: Secret
@@ -221,7 +195,7 @@ tests:
           pattern: my-release-merlin-config$
       - matchRegex:
           path: stringData.[config.yaml]
-          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base-py310:0.0.0-set-from-config"
+          pattern: "ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.38.0-rc1"
       - matchRegex:
           path: stringData.[config.yaml]
-          pattern: "BuildContextURI: git://github.com/caraml-dev/merlin.git#refs/pull/123"
+          pattern: "BuildContextURI: git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1"

charts/merlin/values.yaml

@@ -6,7 +6,7 @@ global:
 # .config when generating templates
 rendered:
   # releasedVersion refers to the git release or tag
-  releasedVersion: v0.35.0
+  releasedVersion: v0.38.0-rc1
   overrides: {}
 
 # set deployment.image.tag to non-nil to overwrite
@@ -266,23 +266,29 @@ imageBuilder:
   #     interactiveMode: IfAvailable
   #     provideClusterInfo: true
   builderConfig:
-    BaseImages:
-      3.8.*:
-        ImageName: pyfunc-py38:v0.1.0
-        DockerfilePath: "docker/Dockerfile"
-        BuildContextURI: "git://github.com/gojek/merlin.git#refs/tags/v0.1"
-        MainAppPath: /merlin-spark-app/main.py
+    ArtifactServiceType: nop
+    BaseImage:
+      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.38.0-rc1
+      DockerfilePath: "pyfunc-server/docker/Dockerfile"
+      BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1"
+      BuildContextSubPath: "python"
     PredictionJobBaseImages:
-      3.8.*:
-        ImageName: pyspark-py38:v0.1.0
-        DockerfilePath: "docker/app.Dockerfile"
-        BuildContextURI: "git://github.com/gojek/merlin.git#refs/tags/v0.1"
-        MainAppPath: /merlin-spark-app/main.py
+      ImageName: ghcr.io/caraml-dev/merlin/merlin-pyspark-base:0.38.0-rc1
+      DockerfilePath: "batch-predictor/docker/app.Dockerfile"
+      BuildContextURI: "git://github.com/caraml-dev/merlin.git#refs/tags/v0.38.0-rc1"
+      BuildContextSubPath: "python"
+      MainAppPath: "/home/spark/merlin-spark-app/main.py"
     BuildNamespace: "mlp"
     DockerRegistry: "dockerRegistry"
     BuildTimeout: "30m"
-    KanikoImage: "gcr.io/kaniko-project/executor:v1.6.0"
-    Resources:
+    KanikoImage: "gcr.io/kaniko-project/executor:v1.18.0"
+    KanikoAdditionalArgs:
+      - "--cache=true"
+      - "--compressed-caching=false"
+      - "--snapshot-mode=redo"
+      - "--use-new-run"
+      - "--log-timestamp"
+    DefaultResources:
       Requests:
         CPU: "1"
         Memory: 512Mi