From 8c038030a61439bb1bb209eb6d20b36afef5d3fe Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:56:32 -0700 Subject: [PATCH 01/11] creating PR for small helm fixes --- helm/etl/Chart.yaml | 2 +- helm/etl/README.md | 2 +- helm/etl/templates/etl-job.yaml | 4 +- helm/fence/Chart.yaml | 2 +- helm/fence/README.md | 2 +- helm/fence/templates/usersync-cron.yaml | 4 +- helm/metadata/Chart.yaml | 2 +- helm/metadata/README.md | 4 +- helm/metadata/templates/metadata.yaml | 10 ----- helm/metadata/values.yaml | 2 +- helm/sower/Chart.yaml | 2 +- helm/sower/README.md | 11 ++++-- helm/sower/templates/_helpers.tpl | 7 ++++ helm/sower/templates/external-secret.yaml | 18 +++++++++ helm/sower/templates/sower-jobs-g3auto.yaml | 10 +++++ helm/sower/values.yaml | 43 ++++++++++++++++++++- 16 files changed, 96 insertions(+), 29 deletions(-) delete mode 100644 helm/metadata/templates/metadata.yaml create mode 100644 helm/sower/templates/sower-jobs-g3auto.yaml diff --git a/helm/etl/Chart.yaml b/helm/etl/Chart.yaml index d55e0945..83ba97e5 100644 --- a/helm/etl/Chart.yaml +++ b/helm/etl/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.6 +version: 0.1.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/etl/README.md b/helm/etl/README.md index 4c1c5995..3ef0e7e9 100644 --- a/helm/etl/README.md +++ b/helm/etl/README.md @@ -1,6 +1,6 @@ # etl -![Version: 0.1.6](https://img.shields.io/badge/Version-0.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.7](https://img.shields.io/badge/Version-0.1.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 etl diff --git a/helm/etl/templates/etl-job.yaml b/helm/etl/templates/etl-job.yaml index 9272940c..34535d57 100644 --- a/helm/etl/templates/etl-job.yaml +++ b/helm/etl/templates/etl-job.yaml @@ -52,7 +52,7 @@ spec: name: etl-mapping - name: fence-yaml configMap: - name: useryaml + name: fence containers: - name: gen3-spark image: {{ .Values.image.spark.repository }}:{{ .Values.image.spark.tag }} @@ -183,7 +183,7 @@ spec: - name: "fence-yaml" readOnly: true mountPath: "/gen3/tube/user.yaml" - subPath: useryaml + subPath: user.yaml resources: requests: cpu: {{ .Values.resources.tube.requests.cpu }} diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index 02930bbf..4df81c7f 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.26 +version: 0.1.27 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/fence/README.md b/helm/fence/README.md index 690caa00..89908be8 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -1,6 +1,6 @@ # fence -![Version: 0.1.26](https://img.shields.io/badge/Version-0.1.26-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.27](https://img.shields.io/badge/Version-0.1.27-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Fence diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml index 2349f8ce..d10ad669 100644 --- a/helm/fence/templates/usersync-cron.yaml +++ b/helm/fence/templates/usersync-cron.yaml @@ -119,8 +119,8 @@ spec: echo "ERROR: failed to generate ETL config" exit 1 fi - # kubectl delete configmap fence > /dev/null 2>&1 - # kubectl create configmap fence --from-file=/tmp/user.yaml + kubectl delete configmap fence > /dev/null 2>&1 + kubectl create configmap fence --from-file=/tmp/user.yaml if [ "${slackWebHook}" != 'None' ]; then curl -X POST --data-urlencode "payload={\"text\": \"AWSHelper: Syncing users on ${gen3Env}\"}" "${slackWebHook}" fi diff --git a/helm/metadata/Chart.yaml b/helm/metadata/Chart.yaml index ecaa6636..02368daa 100644 --- a/helm/metadata/Chart.yaml +++ b/helm/metadata/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.16 +version: 0.1.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/metadata/README.md b/helm/metadata/README.md index e577d5fc..642e62b3 100644 --- a/helm/metadata/README.md +++ b/helm/metadata/README.md @@ -1,6 +1,6 @@ # metadata -![Version: 0.1.16](https://img.shields.io/badge/Version-0.1.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.17](https://img.shields.io/badge/Version-0.1.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Metadata Service @@ -118,5 +118,5 @@ A Helm chart for gen3 Metadata Service | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | -| useAggMds | bool | `"True"` | Set to true to aggregate metadata from multiple other Metadata Service instances. | +| useAggMds | bool | `"False"` | Set to true to aggregate metadata from multiple other Metadata Service instances. | | volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/mds/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. | diff --git a/helm/metadata/templates/metadata.yaml b/helm/metadata/templates/metadata.yaml deleted file mode 100644 index 58700a6e..00000000 --- a/helm/metadata/templates/metadata.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: manifest-metadata -data: - metadata.json: | - { - "USE_AGG_MDS": "{{ .Values.USE_AGG_MDS }}", - "AGG_MDS_NAMESPACE": "{{ .Values.AGG_MDS_NAMESPACE }}" - } \ No newline at end of file diff --git a/helm/metadata/values.yaml b/helm/metadata/values.yaml index 99cbd8ba..fd63e46d 100644 --- a/helm/metadata/values.yaml +++ b/helm/metadata/values.yaml @@ -173,7 +173,7 @@ debug: false # -- (string) Elasticsearch endpoint. esEndpoint: http://gen3-elasticsearch-master:9200 # -- (bool) Set to true to aggregate metadata from multiple other Metadata Service instances. -useAggMds: "True" +useAggMds: "False" # -- (string) Namespae to use if AggMds is enabled. aggMdsNamespace: default diff --git a/helm/sower/Chart.yaml b/helm/sower/Chart.yaml index b47cda9f..c683d22b 100644 --- a/helm/sower/Chart.yaml +++ b/helm/sower/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.15 +version: 0.1.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/sower/README.md b/helm/sower/README.md index 4b883c3c..4d569333 100644 --- a/helm/sower/README.md +++ b/helm/sower/README.md @@ -1,6 +1,6 @@ # sower -![Version: 0.1.15](https://img.shields.io/badge/Version-0.1.15-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.16](https://img.shields.io/badge/Version-0.1.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 sower @@ -31,9 +31,11 @@ A Helm chart for gen3 sower | awsStsRegionalEndpoints | string | `"regional"` | AWS STS to issue temporary credentials to users and roles that make an AWS STS request. Values regional or global. | | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| externalSecrets | map | `{"createK8sPelicanServiceSecret":false,"pelicanserviceG3auto":null}` | External Secrets settings. | -| externalSecrets.createK8sPelicanServiceSecret | string | `false` | Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | +| externalSecrets | map | `{"createK8sPelicanServiceSecret":false,"createK8sSowerJobsSecret":false,"pelicanserviceG3auto":null,"sowerjobsG3auto":null}` | External Secrets settings. | +| externalSecrets.createK8sPelicanServiceSecret | string | `false` | Will create the Helm "pelicanservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | +| externalSecrets.createK8sSowerJobsSecret | string | `false` | Will create the Helm "sower-jobs-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | | externalSecrets.pelicanserviceG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto" | +| externalSecrets.sowerjobsG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "sower-jobs-g3auto" | | fullnameOverride | string | `""` | Override the full name of the deployment. | | gen3Namespace | string | `"default"` | Namespace to deploy the job. | | global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | @@ -45,7 +47,7 @@ A Helm chart for gen3 sower | global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | | global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | | global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. | +| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any sower secrets you have deployed. | | global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | | global.hostname | string | `"localhost"` | Hostname for the deployment. | | global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | @@ -177,6 +179,7 @@ A Helm chart for gen3 sower | sowerConfig[1].restart_policy | string | `"Never"` | | | sowerConfig[1].volumes[0].name | string | `"pelican-creds-volume"` | | | sowerConfig[1].volumes[0].secret.secretName | string | `"pelicanservice-g3auto"` | | +| sowerjobsG3auto | string | `"{\n \"index-object-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\",\n \"indexd_user\": \"diirm\",\n \"indexd_password\": \"$indexdPassword\"\n },\n \"download-indexd-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"get-dbgap-metadata\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"ingest-metadata-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n }\n}\n"` | Additional configuration for Sower Jobs Passed in as a multiline string. | | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | diff --git a/helm/sower/templates/_helpers.tpl b/helm/sower/templates/_helpers.tpl index 1815359e..8f5b72b8 100644 --- a/helm/sower/templates/_helpers.tpl +++ b/helm/sower/templates/_helpers.tpl @@ -72,4 +72,11 @@ Create the name of the service account to use */}} {{- define "pelicanservice-g3auto" -}} {{- default "pelicanservice-g3auto" .Values.externalSecrets.pelicanserviceG3auto }} +{{- end }} + +{{/* + Sowerjobs g3 Auto Secrets Manager Name +*/}} +{{- define "sower-jobs-g3auto" -}} +{{- default "sower-jobs-g3auto" .Values.externalSecrets.sowerjobsG3auto }} {{- end }} \ No newline at end of file diff --git a/helm/sower/templates/external-secret.yaml b/helm/sower/templates/external-secret.yaml index 43132663..d6fd534b 100644 --- a/helm/sower/templates/external-secret.yaml +++ b/helm/sower/templates/external-secret.yaml @@ -1,6 +1,24 @@ {{ if .Values.global.externalSecrets.deploy }} apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret +metadata: + name: pelicanservice-g3auto +spec: + refreshInterval: 5m + secretStoreRef: + name: {{include "common.SecretStore" .}} + kind: SecretStore + target: + name: pelicanservice-g3auto + creationPolicy: Owner + data: + - secretKey: config.json + remoteRef: + #name of secret in secrets manager + key: {{include "pelicanservice-g3auto" .}} +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret metadata: name: pelicanservice-g3auto spec: diff --git a/helm/sower/templates/sower-jobs-g3auto.yaml b/helm/sower/templates/sower-jobs-g3auto.yaml new file mode 100644 index 00000000..0a789e88 --- /dev/null +++ b/helm/sower/templates/sower-jobs-g3auto.yaml @@ -0,0 +1,10 @@ +{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sSowerJobsSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: sower-jobs-g3auto +type: Opaque +stringData: + config.json: | + {{ .Values.sowerjobsG3auto | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/helm/sower/values.yaml b/helm/sower/values.yaml index 8725100a..3b8a41dd 100644 --- a/helm/sower/values.yaml +++ b/helm/sower/values.yaml @@ -55,7 +55,7 @@ global: dispatcherJobNum: "10" # -- (map) External Secrets settings. externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. + # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any sower secrets you have deployed. deploy: false # -- (string) Will deploy a separate External Secret Store for this service. separateSecretStore: false @@ -65,10 +65,14 @@ metricsEnabled: false # -- (map) External Secrets settings. externalSecrets: - # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + # -- (string) Will create the Helm "pelicanservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. createK8sPelicanServiceSecret: false # -- (string) Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto" pelicanserviceG3auto: + # -- (string) Will create the Helm "sower-jobs-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sSowerJobsSecret: false + # -- (string) Will override the name of the aws secrets manager secret. Default is "sower-jobs-g3auto" + sowerjobsG3auto: # -- (map) Secret information for Usersync and External Secrets. secrets: @@ -337,6 +341,41 @@ sowerConfig: secretName: pelicanservice-g3auto restart_policy: Never +# -- (string) Additional configuration for Sower Jobs Passed in as a multiline string. +sowerjobsG3auto: | + { + "index-object-manifest": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName", + "indexd_user": "diirm", + "indexd_password": "$indexdPassword" + }, + "download-indexd-manifest": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName" + }, + "get-dbgap-metadata": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName" + }, + "ingest-metadata-manifest": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName" + } + } + # -- (map) Service account to use or create. serviceAccount: # -- (bool) Specifies whether a service account should be created. From 552bf3b0b016bb98ccc4f57f23480c6e1c0704d7 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:58:48 -0700 Subject: [PATCH 02/11] updating pelican export image to "master" as the code has been fixed to work with Helm --- helm/sower/README.md | 4 ++-- helm/sower/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/sower/README.md b/helm/sower/README.md index 4d569333..daff94c0 100644 --- a/helm/sower/README.md +++ b/helm/sower/README.md @@ -124,7 +124,7 @@ A Helm chart for gen3 sower | sowerConfig[0].container.env[7].name | string | `"SHEEPDOG"` | | | sowerConfig[0].container.env[7].valueFrom.secretKeyRef.key | string | `"sheepdog"` | | | sowerConfig[0].container.env[7].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` | | -| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` | | +| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:master"` | | | sowerConfig[0].container.memory-limit | string | `"12Gi"` | | | sowerConfig[0].container.name | string | `"job-task"` | | | sowerConfig[0].container.pull_policy | string | `"Always"` | | @@ -163,7 +163,7 @@ A Helm chart for gen3 sower | sowerConfig[1].container.env[8].name | string | `"SHEEPDOG"` | | | sowerConfig[1].container.env[8].valueFrom.secretKeyRef.key | string | `"sheepdog"` | | | sowerConfig[1].container.env[8].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` | | -| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` | | +| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:master"` | | | sowerConfig[1].container.memory-limit | string | `"12Gi"` | | | sowerConfig[1].container.name | string | `"job-task"` | | | sowerConfig[1].container.pull_policy | string | `"Always"` | | diff --git a/helm/sower/values.yaml b/helm/sower/values.yaml index 3b8a41dd..55cb4d58 100644 --- a/helm/sower/values.yaml +++ b/helm/sower/values.yaml @@ -226,7 +226,7 @@ sowerConfig: action: export container: name: job-task - image: quay.io/cdis/pelican-export:GPE-1252 + image: quay.io/cdis/pelican-export:master pull_policy: Always env: - name: DICTIONARY_URL @@ -282,7 +282,7 @@ sowerConfig: action: export-files container: name: job-task - image: quay.io/cdis/pelican-export:GPE-1252 + image: quay.io/cdis/pelican-export:master pull_policy: Always env: - name: DICTIONARY_URL From 4328b1eb3f28920663d51f72242b10a4ea102500 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:09:21 -0700 Subject: [PATCH 03/11] fixing gen3 chart versions --- helm/gen3/Chart.yaml | 10 +++++----- helm/gen3/README.md | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 6224603b..d4f39877 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -28,7 +28,7 @@ dependencies: version: 0.1.16 repository: file://../common - name: etl - version: 0.1.6 + version: 0.1.7 repository: file://../etl condition: etl.enabled - name: frontend-framework @@ -36,7 +36,7 @@ dependencies: repository: "file://../frontend-framework" condition: frontend-framework.enabled - name: fence - version: 0.1.26 + version: 0.1.27 repository: "file://../fence" condition: fence.enabled - name: guppy @@ -56,7 +56,7 @@ dependencies: repository: "file://../manifestservice" condition: manifestservice.enabled - name: metadata - version: 0.1.16 + version: 0.1.17 repository: "file://../metadata" condition: metadata.enabled - name: peregrine @@ -84,7 +84,7 @@ dependencies: repository: "file://../ssjdispatcher" condition: ssjdispatcher.enabled - name: sower - version: 0.1.15 + version: 0.1.16 condition: sower.enabled repository: "file://../sower" - name: wts @@ -128,7 +128,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.51 +version: 0.1.52 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 174aacf3..2346ecbf 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,6 +1,6 @@ # gen3 -![Version: 0.1.51](https://img.shields.io/badge/Version-0.1.51-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.52](https://img.shields.io/badge/Version-0.1.52-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons @@ -24,22 +24,22 @@ Helm chart to deploy Gen3 Data Commons | file://../audit | audit | 0.1.16 | | file://../aws-es-proxy | aws-es-proxy | 0.1.13 | | file://../common | common | 0.1.16 | -| file://../etl | etl | 0.1.6 | -| file://../fence | fence | 0.1.26 | +| file://../etl | etl | 0.1.7 | +| file://../fence | fence | 0.1.27 | | file://../frontend-framework | frontend-framework | 0.1.6 | | file://../gen3-network-policies | gen3-network-policies | 0.1.2 | | file://../guppy | guppy | 0.1.16 | | file://../hatchery | hatchery | 0.1.12 | | file://../indexd | indexd | 0.1.18 | | file://../manifestservice | manifestservice | 0.1.17 | -| file://../metadata | metadata | 0.1.16 | +| file://../metadata | metadata | 0.1.17 | | file://../neuvector | neuvector | 0.1.2 | | file://../peregrine | peregrine | 0.1.17 | | file://../portal | portal | 0.1.22 | | file://../requestor | requestor | 0.1.16 | | file://../revproxy | revproxy | 0.1.19 | | file://../sheepdog | sheepdog | 0.1.20 | -| file://../sower | sower | 0.1.15 | +| file://../sower | sower | 0.1.16 | | file://../ssjdispatcher | ssjdispatcher | 0.1.15 | | file://../wts | wts | 0.1.18 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | From 394ba8cf922b5e2567a7a05d24454c9b5b6c43e4 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:35:20 -0700 Subject: [PATCH 04/11] fixing etl environment variable error --- helm/etl/templates/etl-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/etl/templates/etl-job.yaml b/helm/etl/templates/etl-job.yaml index 34535d57..0056c497 100644 --- a/helm/etl/templates/etl-job.yaml +++ b/helm/etl/templates/etl-job.yaml @@ -156,7 +156,7 @@ spec: - name: SPARK_DRIVER_MEMORY value: 6g - name: ETL_FORCED - value: {{ .Values.etlForced }} + value: {{ .Values.etlForced | quote }} - name: gen3Env valueFrom: configMapKeyRef: From fadd6425d856d3330afd79428775ae0a0c1b7c3c Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:35:52 -0700 Subject: [PATCH 05/11] fixing syntax error, updating comment, and removing checksum for deleted configmap. --- helm/metadata/templates/deployment.yaml | 1 - helm/sower/README.md | 2 +- helm/sower/templates/external-secret.yaml | 6 +++--- helm/sower/values.yaml | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml index e4c6ee87..58e06603 100644 --- a/helm/metadata/templates/deployment.yaml +++ b/helm/metadata/templates/deployment.yaml @@ -25,7 +25,6 @@ spec: {{- include "metadata.selectorLabels" . | nindent 8 }} {{- include "common.extraLabels" . | nindent 8 }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/metadata.yaml") . | sha256sum }} {{- if .Values.metricsEnabled }} {{- include "common.grafanaAnnotations" . | nindent 8 }} {{- end }} diff --git a/helm/sower/README.md b/helm/sower/README.md index daff94c0..1c7ea709 100644 --- a/helm/sower/README.md +++ b/helm/sower/README.md @@ -179,7 +179,7 @@ A Helm chart for gen3 sower | sowerConfig[1].restart_policy | string | `"Never"` | | | sowerConfig[1].volumes[0].name | string | `"pelican-creds-volume"` | | | sowerConfig[1].volumes[0].secret.secretName | string | `"pelicanservice-g3auto"` | | -| sowerjobsG3auto | string | `"{\n \"index-object-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\",\n \"indexd_user\": \"diirm\",\n \"indexd_password\": \"$indexdPassword\"\n },\n \"download-indexd-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"get-dbgap-metadata\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"ingest-metadata-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n }\n}\n"` | Additional configuration for Sower Jobs Passed in as a multiline string. | +| sowerjobsG3auto | string | `"{\n \"index-object-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\",\n \"indexd_user\": \"diirm\",\n \"indexd_password\": \"$indexdPassword\"\n },\n \"download-indexd-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"get-dbgap-metadata\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"ingest-metadata-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n }\n}\n"` | Additional configuration for Sower Jobs Passed in as a multiline string. This secret can be mounted in sowerConfig. | | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | diff --git a/helm/sower/templates/external-secret.yaml b/helm/sower/templates/external-secret.yaml index d6fd534b..5296a3d0 100644 --- a/helm/sower/templates/external-secret.yaml +++ b/helm/sower/templates/external-secret.yaml @@ -20,18 +20,18 @@ spec: apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: pelicanservice-g3auto + name: sower-jobs-g3auto spec: refreshInterval: 5m secretStoreRef: name: {{include "common.SecretStore" .}} kind: SecretStore target: - name: pelicanservice-g3auto + name: sower-jobs-g3auto creationPolicy: Owner data: - secretKey: config.json remoteRef: #name of secret in secrets manager - key: {{include "pelicanservice-g3auto" .}} + key: {{include "sower-jobs-g3auto" .}} {{- end }} \ No newline at end of file diff --git a/helm/sower/values.yaml b/helm/sower/values.yaml index 55cb4d58..6459520e 100644 --- a/helm/sower/values.yaml +++ b/helm/sower/values.yaml @@ -341,7 +341,7 @@ sowerConfig: secretName: pelicanservice-g3auto restart_policy: Never -# -- (string) Additional configuration for Sower Jobs Passed in as a multiline string. +# -- (string) Additional configuration for Sower Jobs Passed in as a multiline string. This secret can be mounted in sowerConfig. sowerjobsG3auto: | { "index-object-manifest": { From 564bea2078e7b2813df26fdcee4fc717eb4d9c38 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:49:11 -0700 Subject: [PATCH 06/11] fixing aws-configuration error --- helm/gen3/README.md | 3 +++ helm/gen3/values.yaml | 7 +++++++ helm/sower/README.md | 2 +- helm/sower/values.yaml | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 2346ecbf..6b2ddb3e 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -176,6 +176,9 @@ Helm chart to deploy Gen3 Data Commons | revproxy.ingress.enabled | bool | `false` | Whether to create the custom revproxy ingress | | revproxy.ingress.hosts | list | `[{"host":"chart-example.local"}]` | Where to route the traffic. | | revproxy.ingress.tls | list | `[]` | To secure an Ingress by specifying a secret that contains a TLS private key and certificate. | +| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets and DB Secrets. | +| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | +| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | | sheepdog.enabled | bool | `true` | Whether to deploy the sheepdog subchart. | | ssjdispatcher.enabled | bool | `false` | Whether to deploy the ssjdispatcher subchart. | | wts.enabled | bool | `true` | Whether to deploy the wts subchart. | diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 4caa59a8..52680aaa 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -321,3 +321,10 @@ neuvector: DB_HOST: development-gen3-postgresql # hostname/service name for our ElasitcSearch instance, used to allow egress from containers ES_HOST: gen3-elasticsearch-master + +# -- (map) Secret information for External Secrets and DB Secrets. +secrets: + # -- (str) AWS access key ID. Overrides global key. + awsAccessKeyId: + # -- (str) AWS secret access key ID. Overrides global key. + awsSecretAccessKey: \ No newline at end of file diff --git a/helm/sower/README.md b/helm/sower/README.md index 1c7ea709..3aa7dc4a 100644 --- a/helm/sower/README.md +++ b/helm/sower/README.md @@ -87,7 +87,7 @@ A Helm chart for gen3 sower | resources.requests | map | `{"cpu":"100m","memory":"20Mi"}` | The amount of resources that the container requests | | resources.requests.cpu | string | `"100m"` | The amount of CPU requested | | resources.requests.memory | string | `"20Mi"` | The amount of memory requested | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for Usersync and External Secrets. | +| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Values for sower secrets and keys for External Secrets. | | secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | | secrets.awsSecretAccessKey | str | `nil` | AWS access key ID. Overrides global key. | | securityContext | map | `{}` | Security context for the containers in the pod | diff --git a/helm/sower/values.yaml b/helm/sower/values.yaml index 6459520e..d3b7d5a9 100644 --- a/helm/sower/values.yaml +++ b/helm/sower/values.yaml @@ -74,7 +74,7 @@ externalSecrets: # -- (string) Will override the name of the aws secrets manager secret. Default is "sower-jobs-g3auto" sowerjobsG3auto: -# -- (map) Secret information for Usersync and External Secrets. +# -- (map) Values for sower secrets and keys for External Secrets. secrets: # -- (str) AWS access key ID. Overrides global key. awsAccessKeyId: From 4097aa958aa651f8e7c421a308e3eeec9a2d43f6 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:27:12 -0700 Subject: [PATCH 07/11] bumping gen3 chart version --- helm/gen3/Chart.yaml | 2 +- helm/gen3/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index d4f39877..369452c4 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -128,7 +128,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.52 +version: 0.1.53 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 6b2ddb3e..63b7f507 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,6 +1,6 @@ # gen3 -![Version: 0.1.52](https://img.shields.io/badge/Version-0.1.52-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.53](https://img.shields.io/badge/Version-0.1.53-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons From 4ef7b3a76b762cb77deecdad80be15201cce3a9d Mon Sep 17 00:00:00 2001 From: Ed Date: Mon, 9 Dec 2024 06:48:39 -0600 Subject: [PATCH 08/11] feat(helm-fixes): Added a few more fixes --- helm/gen3/templates/cluster-secret-store.yaml | 6 ++++++ helm/manifestservice/Chart.yaml | 2 +- helm/manifestservice/README.md | 2 +- helm/manifestservice/templates/manifestservice-creds.yaml | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/helm/gen3/templates/cluster-secret-store.yaml b/helm/gen3/templates/cluster-secret-store.yaml index 28ffe29e..38650a4c 100644 --- a/helm/gen3/templates/cluster-secret-store.yaml +++ b/helm/gen3/templates/cluster-secret-store.yaml @@ -12,10 +12,16 @@ spec: region: {{ .Values.global.aws.region }} auth: secretRef: + {{- if .Values.global.aws.useLocalSecret.localSecretName }} accessKeyIDSecretRef: name: {{ .Values.global.aws.useLocalSecret.localSecretName }} key: access-key secretAccessKeySecretRef: name: {{ .Values.global.aws.useLocalSecret.localSecretName }} key: secret-access-key + {{- else }} + jwt: + serviceAccountRef: + name: {{ .Values.global.aws.secretStoreServiceAccount.name }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/manifestservice/Chart.yaml b/helm/manifestservice/Chart.yaml index 84194a1a..cde53eee 100644 --- a/helm/manifestservice/Chart.yaml +++ b/helm/manifestservice/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.17 +version: 0.1.18 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/manifestservice/README.md b/helm/manifestservice/README.md index cc328847..5fef5aa6 100644 --- a/helm/manifestservice/README.md +++ b/helm/manifestservice/README.md @@ -1,6 +1,6 @@ # manifestservice -![Version: 0.1.17](https://img.shields.io/badge/Version-0.1.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.18](https://img.shields.io/badge/Version-0.1.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for Kubernetes diff --git a/helm/manifestservice/templates/manifestservice-creds.yaml b/helm/manifestservice/templates/manifestservice-creds.yaml index 54c5f29a..11fd9990 100644 --- a/helm/manifestservice/templates/manifestservice-creds.yaml +++ b/helm/manifestservice/templates/manifestservice-creds.yaml @@ -9,8 +9,10 @@ stringData: { "manifest_bucket_name": "{{ .Values.manifestserviceG3auto.bucketName }}", "hostname": "{{ .Values.global.hostname }}", + {{ if and .Values.manifestserviceG3auto.awsaccesskey .Values.manifestserviceG3auto.awssecretkey }} "aws_access_key_id": "{{ .Values.manifestserviceG3auto.awsaccesskey }}", "aws_secret_access_key": "{{ .Values.manifestserviceG3auto.awssecretkey }}", + {{ end }} "prefix": "{{ .Values.manifestserviceG3auto.prefix }}" } {{- end }} \ No newline at end of file From f9557c717738fe912f62556a0d1d138e89dfc27c Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 9 Dec 2024 08:21:14 -0700 Subject: [PATCH 09/11] version bump --- helm/gen3/Chart.yaml | 2 +- helm/gen3/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 369452c4..1cf141c7 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -52,7 +52,7 @@ dependencies: repository: "file://../indexd" condition: indexd.enabled - name: manifestservice - version: 0.1.17 + version: 0.1.18 repository: "file://../manifestservice" condition: manifestservice.enabled - name: metadata diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 63b7f507..3d9bbd9c 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -31,7 +31,7 @@ Helm chart to deploy Gen3 Data Commons | file://../guppy | guppy | 0.1.16 | | file://../hatchery | hatchery | 0.1.12 | | file://../indexd | indexd | 0.1.18 | -| file://../manifestservice | manifestservice | 0.1.17 | +| file://../manifestservice | manifestservice | 0.1.18 | | file://../metadata | metadata | 0.1.17 | | file://../neuvector | neuvector | 0.1.2 | | file://../peregrine | peregrine | 0.1.17 | From 41859483e6a2ee0aa7a815360554ea1d400e8896 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:57:45 -0700 Subject: [PATCH 10/11] wrong gen3 version --- helm/gen3/Chart.yaml | 2 +- helm/gen3/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 1cf141c7..84b9ccd7 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -128,7 +128,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.53 +version: 0.1.52 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 3d9bbd9c..a39e4aed 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,6 +1,6 @@ # gen3 -![Version: 0.1.53](https://img.shields.io/badge/Version-0.1.53-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.52](https://img.shields.io/badge/Version-0.1.52-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons From b43378fa3dc476dbe1c6fdae795c4b165d2f2244 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 9 Dec 2024 10:02:44 -0700 Subject: [PATCH 11/11] adding a new line at the end of values.yaml --- helm/gen3/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 52680aaa..05519614 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -327,4 +327,4 @@ secrets: # -- (str) AWS access key ID. Overrides global key. awsAccessKeyId: # -- (str) AWS secret access key ID. Overrides global key. - awsSecretAccessKey: \ No newline at end of file + awsSecretAccessKey: