From 83b501530286edf3b84dff19534bc02a310e798b Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 4 Jul 2024 15:26:47 +0300 Subject: [PATCH 01/10] Add unique identifier for each job run --- charts/fmi-himan/templates/job-template.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/fmi-himan/templates/job-template.yaml b/charts/fmi-himan/templates/job-template.yaml index 2fa712f..31f4746 100644 --- a/charts/fmi-himan/templates/job-template.yaml +++ b/charts/fmi-himan/templates/job-template.yaml @@ -10,7 +10,7 @@ objects: - apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-himan + name: ${RUN_IDENTIFIER} spec: parallelism: 1 completions: 1 @@ -138,6 +138,8 @@ objects: {{- toYaml . | nindent 8 }} {{- end }} parameters: +- description: Unique id for this run + name: RUN_IDENTIFIER - description: Configuration file name name: CONFIGURATION - description: Forecast analysis time, YYYY-MM-DD HH24:MI:SS (optional) From 9a210de40100f382693e6f1d6092b2a94e04d382 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 4 Jul 2024 15:27:50 +0300 Subject: [PATCH 02/10] Cleaner way to use unquoted template parameters --- charts/fmi-himan/templates/job-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/fmi-himan/templates/job-template.yaml b/charts/fmi-himan/templates/job-template.yaml index 31f4746..2e0f88e 100644 --- a/charts/fmi-himan/templates/job-template.yaml +++ b/charts/fmi-himan/templates/job-template.yaml @@ -21,7 +21,7 @@ objects: labels: app.kubernetes.io/name: {{ .Release.Name }}-himan spec: - activeDeadlineSeconds: ${{ "{{" }}TIMEOUT{{ "}}" }} + activeDeadlineSeconds: {{ printf "${{TIMEOUT}}" }} restartPolicy: Never containers: - command: From f5f513ba27fdf05d64e50c760b91c79d7446e4a2 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 4 Jul 2024 15:33:49 +0300 Subject: [PATCH 03/10] Get ecflow rest api host from values --- charts/fmi-himan/templates/job-template.yaml | 6 +++++- charts/fmi-himan/values.yaml | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/fmi-himan/templates/job-template.yaml b/charts/fmi-himan/templates/job-template.yaml index 2e0f88e..c40508d 100644 --- a/charts/fmi-himan/templates/job-template.yaml +++ b/charts/fmi-himan/templates/job-template.yaml @@ -77,7 +77,11 @@ objects: -H 'content-type: application/json' \ -H 'authorization: Bearer testitoken' \ -d "$payload" \ - http://$SNWC_ECFLOW_HTTP_SERVICE_HOST:$SNWC_ECFLOW_HTTP_SERVICE_PORT/v1/suites/$ECF_NAME/attributes + {{- if .Values.ecflow.api.host }} + {{ .Values.ecflow.api.host }}/v1/suites/${ECF_NAME}/attributes + {{- else -}} + http://$SNWC_ECFLOW_HTTP_SERVICE_HOST:$SNWC_ECFLOW_HTTP_SERVICE_PORT/v1/suites/${ECF_NAME}/attributes + {{- end }} fi env: {{ if .Values.s3.credentials.name }} diff --git a/charts/fmi-himan/values.yaml b/charts/fmi-himan/values.yaml index 2114d94..7a21216 100644 --- a/charts/fmi-himan/values.yaml +++ b/charts/fmi-himan/values.yaml @@ -35,6 +35,10 @@ resources: cpu: "1" memory: 1Gi +ecflow: + api: + host: + volumes: [] volumeMounts: [] From bd6fdcca55ddcf521a7a7ad958460d12fbab0ac1 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 4 Jul 2024 15:35:19 +0300 Subject: [PATCH 04/10] Remove pod after 15min the job has executed --- charts/fmi-himan/templates/job-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/fmi-himan/templates/job-template.yaml b/charts/fmi-himan/templates/job-template.yaml index c40508d..f085126 100644 --- a/charts/fmi-himan/templates/job-template.yaml +++ b/charts/fmi-himan/templates/job-template.yaml @@ -15,6 +15,7 @@ objects: parallelism: 1 completions: 1 backoffLimit: 0 + ttlSecondsAfterFinished: 1500 template: metadata: name: {{ .Release.Name }}-himan From 85385ac33bba9f998b2c96ff7af656c6f2b4a858 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 4 Jul 2024 15:38:05 +0300 Subject: [PATCH 05/10] Remove unnecessary whitespace --- charts/fmi-himan/templates/job-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/fmi-himan/templates/job-template.yaml b/charts/fmi-himan/templates/job-template.yaml index f085126..4cbc0e3 100644 --- a/charts/fmi-himan/templates/job-template.yaml +++ b/charts/fmi-himan/templates/job-template.yaml @@ -85,7 +85,7 @@ objects: {{- end }} fi env: - {{ if .Values.s3.credentials.name }} + {{- if .Values.s3.credentials.name }} - name: S3_ACCESS_KEY_ID valueFrom: secretKeyRef: From 0f849e18d04dfdbc1a2978c1fbd4ed45e62f959f Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 4 Jul 2024 15:48:02 +0300 Subject: [PATCH 06/10] Pump up the version --- charts/fmi-himan/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/fmi-himan/Chart.yaml b/charts/fmi-himan/Chart.yaml index 5f27e91..83b9837 100644 --- a/charts/fmi-himan/Chart.yaml +++ b/charts/fmi-himan/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 +version: 1.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 9c72f23b53c599971188b01b1f0808fa37db8ce2 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 4 Jul 2024 16:05:29 +0300 Subject: [PATCH 07/10] Add maintainer --- charts/fmi-himan/Chart.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/fmi-himan/Chart.yaml b/charts/fmi-himan/Chart.yaml index 83b9837..f762368 100644 --- a/charts/fmi-himan/Chart.yaml +++ b/charts/fmi-himan/Chart.yaml @@ -22,3 +22,6 @@ version: 1.0.1 # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. appVersion: "1.0.0" + +maintainers: + - name: osmundi From 2f9b6c4deefb4fcfeca699f6aea9d5ccc903c244 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Mon, 19 Aug 2024 13:42:02 +0300 Subject: [PATCH 08/10] Rerun pipeline --- charts/fmi-himan/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/fmi-himan/Chart.yaml b/charts/fmi-himan/Chart.yaml index f762368..d46c77c 100644 --- a/charts/fmi-himan/Chart.yaml +++ b/charts/fmi-himan/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.1 +version: 1.0.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 24c158288527b2f13ff746b2e70d6ee0cb4728a2 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Thu, 12 Sep 2024 13:35:57 +0300 Subject: [PATCH 09/10] Rebuild himan after the configuration in external git has changed --- charts/fmi-himan/templates/buildconfig.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/fmi-himan/templates/buildconfig.yaml b/charts/fmi-himan/templates/buildconfig.yaml index fee9dfd..46fb6b6 100644 --- a/charts/fmi-himan/templates/buildconfig.yaml +++ b/charts/fmi-himan/templates/buildconfig.yaml @@ -53,4 +53,9 @@ spec: successfulBuildsHistoryLimit: 2 triggers: - type: ConfigChange + - type: Generic + generic: + secretReference: + name: himan-build-trigger-secret + allowEnv: true {{- end }} From 4eba7f89963974a8c2c13afb5546d85f687507c6 Mon Sep 17 00:00:00 2001 From: Ossi Laine Date: Tue, 17 Sep 2024 13:33:23 +0300 Subject: [PATCH 10/10] Secrets from mounted store if installing on aws --- charts/fmi-himan/templates/job-template.yaml | 15 ++++++++++- charts/fmi-himan/templates/secretstorage.yaml | 26 +++++++++++++++++++ charts/fmi-himan/values.yaml | 1 + 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 charts/fmi-himan/templates/secretstorage.yaml diff --git a/charts/fmi-himan/templates/job-template.yaml b/charts/fmi-himan/templates/job-template.yaml index 4cbc0e3..5fc840e 100644 --- a/charts/fmi-himan/templates/job-template.yaml +++ b/charts/fmi-himan/templates/job-template.yaml @@ -134,14 +134,27 @@ objects: requests: cpu: ${CPU_REQUEST} memory: ${MEMORY_REQUEST} - {{- with .Values.volumeMounts }} volumeMounts: + {{- with .Values.volumeMounts }} {{- toYaml . | nindent 10 }} {{- end }} + {{ if eq .Values.cloud "aws" -}} + - name: secrets-store-inline + mountPath: {{ .Values.s3.credentials.storePath }} + readOnly: true + {{- end }} volumes: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{ if eq .Values.cloud "aws" -}} + - name: secrets-store-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ .Release.Name }}-secrets + {{- end }} parameters: - description: Unique id for this run name: RUN_IDENTIFIER diff --git a/charts/fmi-himan/templates/secretstorage.yaml b/charts/fmi-himan/templates/secretstorage.yaml new file mode 100644 index 0000000..113a67b --- /dev/null +++ b/charts/fmi-himan/templates/secretstorage.yaml @@ -0,0 +1,26 @@ +{{ if eq .Values.cloud "aws" -}} +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: {{ .Release.Name }}-secrets + namespace: {{ .Release.Namespace }} +spec: + provider: aws + parameters: + objects: | + - objectName: {{ .Values.s3.credentials.name }} + objectType: secretsmanager + jmesPath: + - path: S3_ACCESS_KEY_ID + objectAlias: accessKey + - path: S3_SECRET_ACCESS_KEY + objectAlias: secretAccessKey + secretObjects: # map secrets from aws secret storage to kubernetes secrets + - data: + - key: S3_ACCESS_KEY_ID + objectName: accessKey + - key: S3_SECRET_ACCESS_KEY + objectName: secretAccessKey + secretName: {{ .Values.s3.credentials.name }} + type: Opaque +{{- end }} diff --git a/charts/fmi-himan/values.yaml b/charts/fmi-himan/values.yaml index 7a21216..4572b8c 100644 --- a/charts/fmi-himan/values.yaml +++ b/charts/fmi-himan/values.yaml @@ -19,6 +19,7 @@ image: s3: credentials: name: + storePath: /mnt/secretstore radon: databaseName: radon