diff --git a/.secrets.baseline b/.secrets.baseline index 51a33818..5a76599f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-04-02T20:57:28Z", + "generated_at": "2024-04-18T16:22:22Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -654,7 +654,7 @@ "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", "is_secret": false, "is_verified": false, - "line_number": 13, + "line_number": 14, "type": "Secret Keyword" } ], diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index e65d4f31..7a4107f9 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -84,7 +84,7 @@ dependencies: repository: "file://../ssjdispatcher" condition: ssjdispatcher.enabled - name: sower - version: 0.1.9 + version: 0.1.10 condition: sower.enabled repository: "file://../sower" - name: wts @@ -115,7 +115,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.29 +version: 0.1.30 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 1e590c50..21adf9b9 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,6 +1,6 @@ # gen3 -![Version: 0.1.29](https://img.shields.io/badge/Version-0.1.29-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.30](https://img.shields.io/badge/Version-0.1.30-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons @@ -37,7 +37,7 @@ Helm chart to deploy Gen3 Data Commons | file://../requestor | requestor | 0.1.10 | | file://../revproxy | revproxy | 0.1.13 | | file://../sheepdog | sheepdog | 0.1.13 | -| file://../sower | sower | 0.1.9 | +| file://../sower | sower | 0.1.10 | | file://../ssjdispatcher | ssjdispatcher | 0.1.8 | | file://../wts | wts | 0.1.12 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | diff --git a/helm/sower/Chart.yaml b/helm/sower/Chart.yaml index 2efaa70d..b273c98b 100644 --- a/helm/sower/Chart.yaml +++ b/helm/sower/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.9 +version: 0.1.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/sower/README.md b/helm/sower/README.md index 2ca77e81..c2a050f4 100644 --- a/helm/sower/README.md +++ b/helm/sower/README.md @@ -1,6 +1,6 @@ # sower -![Version: 0.1.9](https://img.shields.io/badge/Version-0.1.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.10](https://img.shields.io/badge/Version-0.1.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 sower @@ -31,6 +31,9 @@ A Helm chart for gen3 sower | awsStsRegionalEndpoints | string | `"regional"` | AWS STS to issue temporary credentials to users and roles that make an AWS STS request. Values regional or global. | | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | +| externalSecrets | map | `{"createK8sPelicanServiceSecret":false,"pelicanserviceG3auto":null}` | External Secrets settings. | +| externalSecrets.createK8sPelicanServiceSecret | string | `false` | Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | +| externalSecrets.pelicanserviceG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto" | | fullnameOverride | string | `""` | Override the full name of the deployment. | | gen3Namespace | string | `"default"` | Namespace to deploy the job. | | global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | @@ -42,6 +45,9 @@ A Helm chart for gen3 sower | global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | | global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | | global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | +| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | +| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. | +| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | | global.hostname | string | `"localhost"` | Hostname for the deployment. | | global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | | global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | @@ -76,6 +82,9 @@ A Helm chart for gen3 sower | resources.requests | map | `{"cpu":"100m","memory":"20Mi"}` | The amount of resources that the container requests | | resources.requests.cpu | string | `"100m"` | The amount of CPU requested | | resources.requests.memory | string | `"20Mi"` | The amount of memory requested | +| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for Usersync and External Secrets. | +| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | +| secrets.awsSecretAccessKey | str | `nil` | AWS access key ID. Overrides global key. | | securityContext | map | `{}` | Security context for the containers in the pod | | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | | service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. | @@ -95,7 +104,22 @@ A Helm chart for gen3 sower | sowerConfig[0].container.env[1].valueFrom.configMapKeyRef.name | string | `"manifest-global"` | | | sowerConfig[0].container.env[2].name | string | `"ROOT_NODE"` | | | sowerConfig[0].container.env[2].value | string | `"subject"` | | -| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:master"` | | +| sowerConfig[0].container.env[3].name | string | `"DB_HOST"` | | +| sowerConfig[0].container.env[3].valueFrom.secretKeyRef.key | string | `"host"` | | +| sowerConfig[0].container.env[3].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[0].container.env[4].name | string | `"DB_DATABASE"` | | +| sowerConfig[0].container.env[4].valueFrom.secretKeyRef.key | string | `"database"` | | +| sowerConfig[0].container.env[4].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[0].container.env[5].name | string | `"DB_USER"` | | +| sowerConfig[0].container.env[5].valueFrom.secretKeyRef.key | string | `"username"` | | +| sowerConfig[0].container.env[5].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[0].container.env[6].name | string | `"DB_PASS"` | | +| sowerConfig[0].container.env[6].valueFrom.secretKeyRef.key | string | `"password"` | | +| sowerConfig[0].container.env[6].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[0].container.env[7].name | string | `"SHEEPDOG"` | | +| sowerConfig[0].container.env[7].valueFrom.secretKeyRef.key | string | `"sheepdog"` | | +| sowerConfig[0].container.env[7].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` | | +| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` | | | sowerConfig[0].container.memory-limit | string | `"12Gi"` | | | sowerConfig[0].container.name | string | `"job-task"` | | | sowerConfig[0].container.pull_policy | string | `"Always"` | | @@ -103,16 +127,10 @@ A Helm chart for gen3 sower | sowerConfig[0].container.volumeMounts[0].name | string | `"pelican-creds-volume"` | | | sowerConfig[0].container.volumeMounts[0].readOnly | bool | `true` | | | sowerConfig[0].container.volumeMounts[0].subPath | string | `"config.json"` | | -| sowerConfig[0].container.volumeMounts[1].mountPath | string | `"/peregrine-creds.json"` | | -| sowerConfig[0].container.volumeMounts[1].name | string | `"peregrine-creds-volume"` | | -| sowerConfig[0].container.volumeMounts[1].readOnly | bool | `true` | | -| sowerConfig[0].container.volumeMounts[1].subPath | string | `"creds.json"` | | | sowerConfig[0].name | string | `"pelican-export"` | | | sowerConfig[0].restart_policy | string | `"Never"` | | | sowerConfig[0].volumes[0].name | string | `"pelican-creds-volume"` | | | sowerConfig[0].volumes[0].secret.secretName | string | `"pelicanservice-g3auto"` | | -| sowerConfig[0].volumes[1].name | string | `"peregrine-creds-volume"` | | -| sowerConfig[0].volumes[1].secret.secretName | string | `"peregrine-creds"` | | | sowerConfig[1].action | string | `"export-files"` | | | sowerConfig[1].container.cpu-limit | string | `"1"` | | | sowerConfig[1].container.env[0].name | string | `"DICTIONARY_URL"` | | @@ -125,7 +143,22 @@ A Helm chart for gen3 sower | sowerConfig[1].container.env[2].value | string | `"file"` | | | sowerConfig[1].container.env[3].name | string | `"EXTRA_NODES"` | | | sowerConfig[1].container.env[3].value | string | `""` | | -| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:master"` | | +| sowerConfig[1].container.env[4].name | string | `"DB_HOST"` | | +| sowerConfig[1].container.env[4].valueFrom.secretKeyRef.key | string | `"host"` | | +| sowerConfig[1].container.env[4].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[1].container.env[5].name | string | `"DB_DATABASE"` | | +| sowerConfig[1].container.env[5].valueFrom.secretKeyRef.key | string | `"database"` | | +| sowerConfig[1].container.env[5].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[1].container.env[6].name | string | `"DB_USER"` | | +| sowerConfig[1].container.env[6].valueFrom.secretKeyRef.key | string | `"username"` | | +| sowerConfig[1].container.env[6].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[1].container.env[7].name | string | `"DB_PASS"` | | +| sowerConfig[1].container.env[7].valueFrom.secretKeyRef.key | string | `"password"` | | +| sowerConfig[1].container.env[7].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` | | +| sowerConfig[1].container.env[8].name | string | `"SHEEPDOG"` | | +| sowerConfig[1].container.env[8].valueFrom.secretKeyRef.key | string | `"sheepdog"` | | +| sowerConfig[1].container.env[8].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` | | +| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` | | | sowerConfig[1].container.memory-limit | string | `"12Gi"` | | | sowerConfig[1].container.name | string | `"job-task"` | | | sowerConfig[1].container.pull_policy | string | `"Always"` | | @@ -141,8 +174,6 @@ A Helm chart for gen3 sower | sowerConfig[1].restart_policy | string | `"Never"` | | | sowerConfig[1].volumes[0].name | string | `"pelican-creds-volume"` | | | sowerConfig[1].volumes[0].secret.secretName | string | `"pelicanservice-g3auto"` | | -| sowerConfig[1].volumes[1].name | string | `"peregrine-creds-volume"` | | -| sowerConfig[1].volumes[1].secret.secretName | string | `"peregrine-creds"` | | | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | diff --git a/helm/sower/templates/_helpers.tpl b/helm/sower/templates/_helpers.tpl index e9a7c298..1815359e 100644 --- a/helm/sower/templates/_helpers.tpl +++ b/helm/sower/templates/_helpers.tpl @@ -66,3 +66,10 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* + Pelicanservice g3 Auto Secrets Manager Name +*/}} +{{- define "pelicanservice-g3auto" -}} +{{- default "pelicanservice-g3auto" .Values.externalSecrets.pelicanserviceG3auto }} +{{- end }} \ No newline at end of file diff --git a/helm/sower/templates/aws-config.yaml b/helm/sower/templates/aws-config.yaml new file mode 100644 index 00000000..398770d3 --- /dev/null +++ b/helm/sower/templates/aws-config.yaml @@ -0,0 +1,3 @@ +{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} +{{ include "common.awsconfig" . }} +{{- end -}} \ No newline at end of file diff --git a/helm/sower/templates/external-secret.yaml b/helm/sower/templates/external-secret.yaml new file mode 100644 index 00000000..43132663 --- /dev/null +++ b/helm/sower/templates/external-secret.yaml @@ -0,0 +1,19 @@ +{{ if .Values.global.externalSecrets.deploy }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: pelicanservice-g3auto +spec: + refreshInterval: 5m + secretStoreRef: + name: {{include "common.SecretStore" .}} + kind: SecretStore + target: + name: pelicanservice-g3auto + creationPolicy: Owner + data: + - secretKey: config.json + remoteRef: + #name of secret in secrets manager + key: {{include "pelicanservice-g3auto" .}} +{{- end }} \ No newline at end of file diff --git a/helm/sower/templates/pelican-creds.yaml b/helm/sower/templates/pelican-creds.yaml index d4ab607f..0d3420f5 100644 --- a/helm/sower/templates/pelican-creds.yaml +++ b/helm/sower/templates/pelican-creds.yaml @@ -1,3 +1,4 @@ +{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sPelicanServiceSecret) }} apiVersion: v1 kind: Secret metadata: @@ -9,7 +10,8 @@ stringData: { "manifest_bucket_name": "{{ .Values.pelican.bucket }}", "hostname": "{{ .Values.global.hostname }}", - "aws_access_key_id": "{{ .Values.global.aws.pelican_user.access_key }}", - "aws_secret_access_key": "{{ .Values.global.aws.pelican_user.access_secret }}" + "aws_access_key_id": "{{ .Values.secrets.awsAccessKeyId | default .Values.global.aws.awsAccessKeyId }}", + "aws_secret_access_key": "{{ .Values.secrets.awsSecretAccessKey | default .Values.global.aws.awsSecretAccessKey }}" } +{{- end }} {{- end }} \ No newline at end of file diff --git a/helm/sower/templates/secret-store.yaml b/helm/sower/templates/secret-store.yaml new file mode 100644 index 00000000..771c7760 --- /dev/null +++ b/helm/sower/templates/secret-store.yaml @@ -0,0 +1,3 @@ +{{ if .Values.global.externalSecrets.separateSecretStore }} +{{ include "common.secretstore" . }} +{{- end }} \ No newline at end of file diff --git a/helm/sower/values.yaml b/helm/sower/values.yaml index 3bc59048..48f36db2 100644 --- a/helm/sower/values.yaml +++ b/helm/sower/values.yaml @@ -55,6 +55,26 @@ global: dispatcherJobNum: "10" # -- (bool) Whether Datadog is enabled. ddEnabled: false + # -- (map) External Secrets settings. + externalSecrets: + # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. + deploy: false + # -- (string) Will deploy a separate External Secret Store for this service. + separateSecretStore: false + +# -- (map) External Secrets settings. +externalSecrets: + # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sPelicanServiceSecret: false + # -- (string) Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto" + pelicanserviceG3auto: + +# -- (map) Secret information for Usersync and External Secrets. +secrets: + # -- (str) AWS access key ID. Overrides global key. + awsAccessKeyId: + # -- (str) AWS access key ID. Overrides global key. + awsSecretAccessKey: # -- (int) Number of replicas for the deployment. replicaCount: 1 @@ -190,7 +210,7 @@ sowerConfig: action: export container: name: job-task - image: quay.io/cdis/pelican-export:master + image: quay.io/cdis/pelican-export:GPE-1252 pull_policy: Always env: - name: DICTIONARY_URL @@ -205,30 +225,48 @@ sowerConfig: key: hostname - name: ROOT_NODE value: subject + - name: DB_HOST + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: host + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: database + - name: DB_USER + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: username + - name: DB_PASS + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: password + - name: SHEEPDOG + valueFrom: + secretKeyRef: + name: indexd-service-creds + key: sheepdog volumeMounts: - name: pelican-creds-volume readOnly: true mountPath: "/pelican-creds.json" subPath: config.json - - name: peregrine-creds-volume - readOnly: true - mountPath: "/peregrine-creds.json" - subPath: creds.json cpu-limit: '1' memory-limit: 12Gi volumes: - name: pelican-creds-volume secret: secretName: pelicanservice-g3auto - - name: peregrine-creds-volume - secret: - secretName: peregrine-creds restart_policy: Never - name: pelican-export-files action: export-files container: name: job-task - image: quay.io/cdis/pelican-export:master + image: quay.io/cdis/pelican-export:GPE-1252 pull_policy: Always env: - name: DICTIONARY_URL @@ -245,6 +283,31 @@ sowerConfig: value: file - name: EXTRA_NODES value: '' + - name: DB_HOST + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: host + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: database + - name: DB_USER + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: username + - name: DB_PASS + valueFrom: + secretKeyRef: + name: peregrine-dbcreds + key: password + - name: SHEEPDOG + valueFrom: + secretKeyRef: + name: indexd-service-creds + key: sheepdog volumeMounts: - name: pelican-creds-volume readOnly: true @@ -260,9 +323,6 @@ sowerConfig: - name: pelican-creds-volume secret: secretName: pelicanservice-g3auto - - name: peregrine-creds-volume - secret: - secretName: peregrine-creds restart_policy: Never