Merge pull request #169 from uc-cdis/feat/external-secrets-additions

External Secrets Updates for Sower

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-04-02T20:57:28Z",
+  "generated_at": "2024-04-18T16:22:22Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -654,7 +654,7 @@
         "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 13,
+        "line_number": 14,
         "type": "Secret Keyword"
       }
     ],

helm/gen3/Chart.yaml

@@ -84,7 +84,7 @@ dependencies:
   repository: "file://../ssjdispatcher"
   condition: ssjdispatcher.enabled
 - name: sower
-  version: 0.1.9
+  version: 0.1.10
   condition: sower.enabled
   repository: "file://../sower"
 - name: wts
@@ -115,7 +115,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.29
+version: 0.1.30
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/gen3/README.md

@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.29](https://img.shields.io/badge/Version-0.1.29-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.30](https://img.shields.io/badge/Version-0.1.30-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -37,7 +37,7 @@ Helm chart to deploy Gen3 Data Commons
 | file://../requestor | requestor | 0.1.10 |
 | file://../revproxy | revproxy | 0.1.13 |
 | file://../sheepdog | sheepdog | 0.1.13 |
-| file://../sower | sower | 0.1.9 |
+| file://../sower | sower | 0.1.10 |
 | file://../ssjdispatcher | ssjdispatcher | 0.1.8 |
 | file://../wts | wts | 0.1.12 |
 | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 |

helm/sower/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.9
+version: 0.1.10
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/sower/README.md

@@ -1,6 +1,6 @@
 # sower
 
-![Version: 0.1.9](https://img.shields.io/badge/Version-0.1.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.10](https://img.shields.io/badge/Version-0.1.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 sower
 
@@ -31,6 +31,9 @@ A Helm chart for gen3 sower
 | awsStsRegionalEndpoints | string | `"regional"` | AWS STS to issue temporary credentials to users and roles that make an AWS STS request. Values regional or global. |
 | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl |
 | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". |
+| externalSecrets | map | `{"createK8sPelicanServiceSecret":false,"pelicanserviceG3auto":null}` | External Secrets settings. |
+| externalSecrets.createK8sPelicanServiceSecret | string | `false` | Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. |
+| externalSecrets.pelicanserviceG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto" |
 | fullnameOverride | string | `""` | Override the full name of the deployment. |
 | gen3Namespace | string | `"default"` | Namespace to deploy the job. |
 | global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration |
@@ -42,6 +45,9 @@ A Helm chart for gen3 sower
 | global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. |
 | global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. |
 | global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. |
+| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. |
+| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. |
+| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. |
 | global.hostname | string | `"localhost"` | Hostname for the deployment. |
 | global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. |
 | global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. |
@@ -76,6 +82,9 @@ A Helm chart for gen3 sower
 | resources.requests | map | `{"cpu":"100m","memory":"20Mi"}` | The amount of resources that the container requests |
 | resources.requests.cpu | string | `"100m"` | The amount of CPU requested |
 | resources.requests.memory | string | `"20Mi"` | The amount of memory requested |
+| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for Usersync and External Secrets. |
+| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. |
+| secrets.awsSecretAccessKey | str | `nil` | AWS access key ID. Overrides global key. |
 | securityContext | map | `{}` | Security context for the containers in the pod |
 | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl |
 | service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. |
@@ -95,24 +104,33 @@ A Helm chart for gen3 sower
 | sowerConfig[0].container.env[1].valueFrom.configMapKeyRef.name | string | `"manifest-global"` |  |
 | sowerConfig[0].container.env[2].name | string | `"ROOT_NODE"` |  |
 | sowerConfig[0].container.env[2].value | string | `"subject"` |  |
-| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:master"` |  |
+| sowerConfig[0].container.env[3].name | string | `"DB_HOST"` |  |
+| sowerConfig[0].container.env[3].valueFrom.secretKeyRef.key | string | `"host"` |  |
+| sowerConfig[0].container.env[3].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[0].container.env[4].name | string | `"DB_DATABASE"` |  |
+| sowerConfig[0].container.env[4].valueFrom.secretKeyRef.key | string | `"database"` |  |
+| sowerConfig[0].container.env[4].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[0].container.env[5].name | string | `"DB_USER"` |  |
+| sowerConfig[0].container.env[5].valueFrom.secretKeyRef.key | string | `"username"` |  |
+| sowerConfig[0].container.env[5].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[0].container.env[6].name | string | `"DB_PASS"` |  |
+| sowerConfig[0].container.env[6].valueFrom.secretKeyRef.key | string | `"password"` |  |
+| sowerConfig[0].container.env[6].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[0].container.env[7].name | string | `"SHEEPDOG"` |  |
+| sowerConfig[0].container.env[7].valueFrom.secretKeyRef.key | string | `"sheepdog"` |  |
+| sowerConfig[0].container.env[7].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` |  |
+| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` |  |
 | sowerConfig[0].container.memory-limit | string | `"12Gi"` |  |
 | sowerConfig[0].container.name | string | `"job-task"` |  |
 | sowerConfig[0].container.pull_policy | string | `"Always"` |  |
 | sowerConfig[0].container.volumeMounts[0].mountPath | string | `"/pelican-creds.json"` |  |
 | sowerConfig[0].container.volumeMounts[0].name | string | `"pelican-creds-volume"` |  |
 | sowerConfig[0].container.volumeMounts[0].readOnly | bool | `true` |  |
 | sowerConfig[0].container.volumeMounts[0].subPath | string | `"config.json"` |  |
-| sowerConfig[0].container.volumeMounts[1].mountPath | string | `"/peregrine-creds.json"` |  |
-| sowerConfig[0].container.volumeMounts[1].name | string | `"peregrine-creds-volume"` |  |
-| sowerConfig[0].container.volumeMounts[1].readOnly | bool | `true` |  |
-| sowerConfig[0].container.volumeMounts[1].subPath | string | `"creds.json"` |  |
 | sowerConfig[0].name | string | `"pelican-export"` |  |
 | sowerConfig[0].restart_policy | string | `"Never"` |  |
 | sowerConfig[0].volumes[0].name | string | `"pelican-creds-volume"` |  |
 | sowerConfig[0].volumes[0].secret.secretName | string | `"pelicanservice-g3auto"` |  |
-| sowerConfig[0].volumes[1].name | string | `"peregrine-creds-volume"` |  |
-| sowerConfig[0].volumes[1].secret.secretName | string | `"peregrine-creds"` |  |
 | sowerConfig[1].action | string | `"export-files"` |  |
 | sowerConfig[1].container.cpu-limit | string | `"1"` |  |
 | sowerConfig[1].container.env[0].name | string | `"DICTIONARY_URL"` |  |
@@ -125,7 +143,22 @@ A Helm chart for gen3 sower
 | sowerConfig[1].container.env[2].value | string | `"file"` |  |
 | sowerConfig[1].container.env[3].name | string | `"EXTRA_NODES"` |  |
 | sowerConfig[1].container.env[3].value | string | `""` |  |
-| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:master"` |  |
+| sowerConfig[1].container.env[4].name | string | `"DB_HOST"` |  |
+| sowerConfig[1].container.env[4].valueFrom.secretKeyRef.key | string | `"host"` |  |
+| sowerConfig[1].container.env[4].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[1].container.env[5].name | string | `"DB_DATABASE"` |  |
+| sowerConfig[1].container.env[5].valueFrom.secretKeyRef.key | string | `"database"` |  |
+| sowerConfig[1].container.env[5].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[1].container.env[6].name | string | `"DB_USER"` |  |
+| sowerConfig[1].container.env[6].valueFrom.secretKeyRef.key | string | `"username"` |  |
+| sowerConfig[1].container.env[6].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[1].container.env[7].name | string | `"DB_PASS"` |  |
+| sowerConfig[1].container.env[7].valueFrom.secretKeyRef.key | string | `"password"` |  |
+| sowerConfig[1].container.env[7].valueFrom.secretKeyRef.name | string | `"peregrine-dbcreds"` |  |
+| sowerConfig[1].container.env[8].name | string | `"SHEEPDOG"` |  |
+| sowerConfig[1].container.env[8].valueFrom.secretKeyRef.key | string | `"sheepdog"` |  |
+| sowerConfig[1].container.env[8].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` |  |
+| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` |  |
 | sowerConfig[1].container.memory-limit | string | `"12Gi"` |  |
 | sowerConfig[1].container.name | string | `"job-task"` |  |
 | sowerConfig[1].container.pull_policy | string | `"Always"` |  |
@@ -141,8 +174,6 @@ A Helm chart for gen3 sower
 | sowerConfig[1].restart_policy | string | `"Never"` |  |
 | sowerConfig[1].volumes[0].name | string | `"pelican-creds-volume"` |  |
 | sowerConfig[1].volumes[0].secret.secretName | string | `"pelicanservice-g3auto"` |  |
-| sowerConfig[1].volumes[1].name | string | `"peregrine-creds-volume"` |  |
-| sowerConfig[1].volumes[1].secret.secretName | string | `"peregrine-creds"` |  |
 | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy |
 | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. |
 | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. |

helm/sower/templates/_helpers.tpl

@@ -66,3 +66,10 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+  Pelicanservice g3 Auto Secrets Manager Name
+*/}}
+{{- define "pelicanservice-g3auto" -}}
+{{- default "pelicanservice-g3auto" .Values.externalSecrets.pelicanserviceG3auto }}
+{{- end }}

helm/sower/templates/aws-config.yaml

@@ -0,0 +1,3 @@
+{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }}
+{{ include "common.awsconfig" . }}
+{{- end -}}

helm/sower/templates/external-secret.yaml

@@ -0,0 +1,19 @@
+{{ if .Values.global.externalSecrets.deploy }}
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: pelicanservice-g3auto
+spec:
+  refreshInterval: 5m
+  secretStoreRef:
+    name: {{include "common.SecretStore" .}}
+    kind: SecretStore
+  target:
+    name: pelicanservice-g3auto
+    creationPolicy: Owner
+  data:
+  - secretKey: config.json
+    remoteRef:
+      #name of secret in secrets manager
+      key: {{include "pelicanservice-g3auto" .}}
+{{- end }}

helm/sower/templates/pelican-creds.yaml

@@ -1,3 +1,4 @@
+{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sPelicanServiceSecret) }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -9,7 +10,8 @@ stringData:
     {
       "manifest_bucket_name": "{{ .Values.pelican.bucket }}",
       "hostname": "{{ .Values.global.hostname }}",
-      "aws_access_key_id": "{{ .Values.global.aws.pelican_user.access_key }}",
-      "aws_secret_access_key": "{{ .Values.global.aws.pelican_user.access_secret }}"
+      "aws_access_key_id": "{{ .Values.secrets.awsAccessKeyId | default .Values.global.aws.awsAccessKeyId }}",
+      "aws_secret_access_key": "{{ .Values.secrets.awsSecretAccessKey | default .Values.global.aws.awsSecretAccessKey }}"
     }
+{{- end }}
 {{- end }}

helm/sower/templates/secret-store.yaml

@@ -0,0 +1,3 @@
+{{ if .Values.global.externalSecrets.separateSecretStore }}
+{{ include "common.secretstore" . }}
+{{- end  }}

helm/sower/values.yaml

@@ -55,6 +55,26 @@ global:
   dispatcherJobNum: "10"
   # -- (bool) Whether Datadog is enabled.
   ddEnabled: false
+  # -- (map) External Secrets settings.
+  externalSecrets:
+    # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed.
+    deploy: false
+    # -- (string) Will deploy a separate External Secret Store for this service.
+    separateSecretStore: false
+
+# -- (map) External Secrets settings.
+externalSecrets:
+  # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets.
+  createK8sPelicanServiceSecret: false
+  # -- (string) Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto"
+  pelicanserviceG3auto:
+
+# -- (map) Secret information for Usersync and External Secrets.
+secrets:
+  # -- (str) AWS access key ID. Overrides global key.
+  awsAccessKeyId:
+  # -- (str) AWS access key ID. Overrides global key.
+  awsSecretAccessKey:
 
 # -- (int) Number of replicas for the deployment.
 replicaCount: 1
@@ -190,7 +210,7 @@ sowerConfig:
     action: export
     container:
       name: job-task
-      image: quay.io/cdis/pelican-export:master
+      image: quay.io/cdis/pelican-export:GPE-1252
       pull_policy: Always
       env:
       - name: DICTIONARY_URL
@@ -205,30 +225,48 @@ sowerConfig:
             key: hostname
       - name: ROOT_NODE
         value: subject
+      - name: DB_HOST
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: host
+      - name: DB_DATABASE
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: database
+      - name: DB_USER
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: username
+      - name: DB_PASS
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: password
+      - name: SHEEPDOG
+        valueFrom:
+          secretKeyRef:
+            name: indexd-service-creds
+            key: sheepdog
       volumeMounts:
       - name: pelican-creds-volume
         readOnly: true
         mountPath: "/pelican-creds.json"
         subPath: config.json
-      - name: peregrine-creds-volume
-        readOnly: true
-        mountPath: "/peregrine-creds.json"
-        subPath: creds.json
       cpu-limit: '1'
       memory-limit: 12Gi
     volumes:
     - name: pelican-creds-volume
       secret:
         secretName: pelicanservice-g3auto
-    - name: peregrine-creds-volume
-      secret:
-        secretName: peregrine-creds
     restart_policy: Never
   - name: pelican-export-files
     action: export-files
     container:
       name: job-task
-      image: quay.io/cdis/pelican-export:master
+      image: quay.io/cdis/pelican-export:GPE-1252
       pull_policy: Always
       env:
       - name: DICTIONARY_URL
@@ -245,6 +283,31 @@ sowerConfig:
         value: file
       - name: EXTRA_NODES
         value: ''
+      - name: DB_HOST
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: host
+      - name: DB_DATABASE
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: database
+      - name: DB_USER
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: username
+      - name: DB_PASS
+        valueFrom:
+          secretKeyRef:
+            name: peregrine-dbcreds
+            key: password
+      - name: SHEEPDOG
+        valueFrom:
+          secretKeyRef:
+            name: indexd-service-creds
+            key: sheepdog
       volumeMounts:
       - name: pelican-creds-volume
         readOnly: true
@@ -260,9 +323,6 @@ sowerConfig:
     - name: pelican-creds-volume
       secret:
         secretName: pelicanservice-g3auto
-    - name: peregrine-creds-volume
-      secret:
-        secretName: peregrine-creds
     restart_policy: Never