diff --git a/.gitignore b/.gitignore index acfc0957..fad70505 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ postgres.txt +**/charts/ +values.yaml \ No newline at end of file diff --git a/.secrets.baseline b/.secrets.baseline index 0607e230..b9b20b03 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2022-09-19T14:03:53Z", + "generated_at": "2022-10-24T21:24:09Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -74,58 +74,90 @@ "type": "Secret Keyword" } ], - "helm/arborist/values.yaml": [ + "helm/audit/templates/secrets.yaml": [ { - "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", + "hashed_secret": "57d8659e51a2da40fda65f6a4cc294cc3f59cf6d", "is_secret": false, "is_verified": false, - "line_number": 116, + "line_number": 25, "type": "Secret Keyword" } ], - "helm/audit/values.yaml": [ + "helm/common/templates/_db_setup_job.tpl": [ { - "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", + "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", "is_secret": false, "is_verified": false, - "line_number": 167, + "line_number": 85, "type": "Secret Keyword" } ], - "helm/db-setup/templates/_db_setup.tpl": [ + "helm/common/templates/_postgres_secrets.tpl": [ + { + "hashed_secret": "07b87392697bbdd9d97f6cd887f901820a0150df", + "is_secret": false, + "is_verified": false, + "line_number": 27, + "type": "Secret Keyword" + }, + { + "hashed_secret": "e343239977fa87adac52528619fc6bf2e1a82ee7", + "is_secret": false, + "is_verified": false, + "line_number": 55, + "type": "Secret Keyword" + }, { "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", + "is_secret": false, "is_verified": false, - "line_number": 24, + "line_number": 92, "type": "Secret Keyword" } ], - "helm/db-setup/values.yaml": [ + "helm/common/templates/_secrets.tpl": [ { - "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", + "hashed_secret": "e540cdd1328b2b21e29a95405c301b9313b7c346", + "is_secret": false, "is_verified": false, - "line_number": 6, + "line_number": 96, "type": "Secret Keyword" }, { - "hashed_secret": "a70646783e43f444ba3430a4110bb7bdd65bdb3a", + "hashed_secret": "67caac52553e052426982b6f096e73318b151765", + "is_secret": false, "is_verified": false, - "line_number": 12, + "line_number": 115, "type": "Secret Keyword" }, { - "hashed_secret": "874947acc1ffd819b836f6e049b2f1ab8303cb6c", + "hashed_secret": "17849dced8de4397e88a8b1c746477aead486a2b", + "is_secret": false, "is_verified": false, - "line_number": 15, + "line_number": 116, + "type": "Secret Keyword" + }, + { + "hashed_secret": "df39b4caf493869772ff3a0f95cca6a9ae7934dc", + "is_secret": false, + "is_verified": false, + "line_number": 117, + "type": "Secret Keyword" + }, + { + "hashed_secret": "07b87392697bbdd9d97f6cd887f901820a0150df", + "is_secret": false, + "is_verified": false, + "line_number": 119, "type": "Secret Keyword" } ], - "helm/fence/fence-config/fence-config.yaml": [ + "helm/dicom-server/values.yaml": [ { - "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db", + "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_secret": false, "is_verified": false, - "line_number": 587, + "line_number": 69, "type": "Secret Keyword" } ], @@ -163,7 +195,7 @@ ], "helm/fence/templates/fence-creds.yaml": [ { - "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", + "hashed_secret": "c2dae5a3c7ce218639b38d8a0256f02fe81d439e", "is_secret": false, "is_verified": false, "line_number": 11, @@ -185,18 +217,11 @@ } ], "helm/fence/values.yaml": [ - { - "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", - "is_secret": false, - "is_verified": false, - "line_number": 29, - "type": "Secret Keyword" - }, { "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db", "is_secret": false, "is_verified": false, - "line_number": 864, + "line_number": 870, "type": "Secret Keyword" } ], @@ -227,12 +252,12 @@ "type": "Basic Auth Credentials" } ], - "helm/indexd/values.yaml": [ + "helm/indexd/templates/indexd-secret.yaml": [ { - "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", + "hashed_secret": "c2dae5a3c7ce218639b38d8a0256f02fe81d439e", "is_secret": false, "is_verified": false, - "line_number": 108, + "line_number": 25, "type": "Secret Keyword" } ], @@ -245,15 +270,6 @@ "type": "Secret Keyword" } ], - "helm/metadata/values.yaml": [ - { - "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", - "is_secret": false, - "is_verified": false, - "line_number": 146, - "type": "Secret Keyword" - } - ], "helm/peregrine/peregrine-secret/config_helper.py": [ { "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", @@ -268,36 +284,29 @@ "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", "is_secret": false, "is_verified": false, - "line_number": 46, + "line_number": 45, "type": "Basic Auth Credentials" } ], - "helm/peregrine/values.yaml": [ - { - "hashed_secret": "c286f6974f94aab4cfaf2ef49ee0465a8495f563", - "is_secret": false, - "is_verified": false, - "line_number": 4, - "type": "Secret Keyword" - }, + "helm/sheepdog/sheepdog-secret/config_helper.py": [ { - "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", + "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", "is_secret": false, "is_verified": false, - "line_number": 195, - "type": "Secret Keyword" + "line_number": 66, + "type": "Basic Auth Credentials" } ], - "helm/sheepdog/sheepdog-secret/config_helper.py": [ + "helm/sheepdog/sheepdog-secret/wsgi.py": [ { - "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", + "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", "is_secret": false, "is_verified": false, - "line_number": 66, + "line_number": 45, "type": "Basic Auth Credentials" } ], - "helm/sheepdog/sheepdog-secret/wsgi.py": [ + "helm/sheepdog/sheepdog-secret/wsgi_copy.py": [ { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", "is_secret": false, @@ -311,7 +320,7 @@ "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_secret": false, "is_verified": false, - "line_number": 132, + "line_number": 150, "type": "Secret Keyword" } ], @@ -341,20 +350,6 @@ "line_number": 29, "type": "Secret Keyword" } - ], - "helm/wts/values.yaml": [ - { - "hashed_secret": "7f8d4ff76d11ef5e9aa633658ed42694b91d3da9", - "is_verified": false, - "line_number": 116, - "type": "Secret Keyword" - }, - { - "hashed_secret": "a70646783e43f444ba3430a4110bb7bdd65bdb3a", - "is_verified": false, - "line_number": 122, - "type": "Secret Keyword" - } ] }, "version": "0.13.1", diff --git a/helm/ambassador/Chart.yaml b/helm/ambassador/Chart.yaml index 18018a4f..65b0542d 100644 --- a/helm/ambassador/Chart.yaml +++ b/helm/ambassador/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: ambassador -description: A Helm chart for Kubernetes +description: A Helm chart for deploying ambassador for gen3 # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "1.4.2" diff --git a/helm/arborist/Chart.lock b/helm/arborist/Chart.lock new file mode 100644 index 00000000..52a7fd5c --- /dev/null +++ b/helm/arborist/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:a25c79b74ec6d89ca5c732e4222f8726ed02aa6a4a21f376afc499e53696c9b5 +generated: "2022-10-20T21:34:32.587406-05:00" diff --git a/helm/arborist/Chart.yaml b/helm/arborist/Chart.yaml index 9d400923..b875d331 100644 --- a/helm/arborist/Chart.yaml +++ b/helm/arborist/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: arborist -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 arborist # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,15 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2022.05" \ No newline at end of file +appVersion: "2022.10" + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common \ No newline at end of file diff --git a/helm/arborist/templates/_helpers.tpl b/helm/arborist/templates/_helpers.tpl index e58113bc..db6153b5 100644 --- a/helm/arborist/templates/_helpers.tpl +++ b/helm/arborist/templates/_helpers.tpl @@ -69,6 +69,6 @@ Create the name of the service account to use {{- if $localpass }} {{- default (index $localpass.data "postgres-password" | b64dec) }} {{- else }} -{{- default .Values.database.password }} +{{- default .Values.postgres.password }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/arborist/templates/arborist-creds.yaml b/helm/arborist/templates/arborist-creds.yaml deleted file mode 100644 index 2df92a4f..00000000 --- a/helm/arborist/templates/arborist-creds.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: arborist-creds -type: Opaque -stringData: - dbcreds.json: |- - { - "db_host": {{ .Values.database.host | quote }}, - "db_username": {{ .Values.database.user | quote}}, - "db_password": {{ include "arborist.postgres.password" . | quote }}, - "db_database": {{ .Values.database.dbname | quote }} - } - - diff --git a/helm/arborist/templates/db-init.yaml b/helm/arborist/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/arborist/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/arborist/templates/deployment.yaml b/helm/arborist/templates/deployment.yaml index a91d1032..26dfb3c5 100644 --- a/helm/arborist/templates/deployment.yaml +++ b/helm/arborist/templates/deployment.yaml @@ -59,14 +59,8 @@ spec: - "-c" - | # set env vars - export PGDATABASE=$(cat /var/www/arborist/dbcreds.json | jq -r '.db_database') - export PGUSER=$(cat /var/www/arborist/dbcreds.json | jq -r '.db_username') - export PGPASSWORD=$(cat /var/www/arborist/dbcreds.json | jq -r '.db_password') - export PGHOST=$(cat /var/www/arborist/dbcreds.json | jq -r '.db_host') - export PGPORT="5432" export PGSSLMODE="disable" - # bring the database schema up to the latest version /go/src/github.com/uc-cdis/arborist/migrations/latest diff --git a/helm/arborist/values.yaml b/helm/arborist/values.yaml index d78b2151..f2314f8e 100644 --- a/helm/arborist/values.yaml +++ b/helm/arborist/values.yaml @@ -1,7 +1,24 @@ - # Default values for arborist. # This is a YAML-formatted file. # Declare variables to be passed into your templates. +global: + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + +db_create: true +postgres: + host: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: arborist + username: arborist + port: 5432 + # If left empty password will be auto-generated + password: + replicaCount: 1 @@ -41,25 +58,6 @@ service: type: ClusterIP port: 80 -ingress: - enabled: true - className: "" - annotations: { - nginx.ingress.kubernetes.io/rewrite-target: /$2, - kubernetes.io/ingress.class: "nginx" - } - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: ingress.local - paths: - - path: /authz(/|$)(.*) - pathType: Prefix - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -89,29 +87,46 @@ affinity: {} volumes: - name: creds-volume secret: - secretName: "arborist-creds" + secretName: "arborist-dbcreds" volumeMounts: -- name: "creds-volume" - readOnly: true - mountPath: "/var/www/arborist/dbcreds.json" - subPath: dbcreds.json +# TODO: REMOVE? env: +# TODO: Revisit this? - name: JWKS_ENDPOINT value: "http://fence-service/.well-known/jwks" +- name: PGPASSWORD + valueFrom: + secretKeyRef: + name: arborist-dbcreds + key: password + optional: false +- name: PGUSER + valueFrom: + secretKeyRef: + name: arborist-dbcreds + key: username + optional: false +- name: PGDATABASE + valueFrom: + secretKeyRef: + name: arborist-dbcreds + key: database + optional: false +- name: PGHOST + valueFrom: + secretKeyRef: + name: arborist-dbcreds + key: host + optional: false +- name: PGPORT + valueFrom: + secretKeyRef: + name: arborist-dbcreds + key: port + optional: false +- name: PGSSLMODE + value: disable - -database: - port: 5432 - host: postgres-postgresql.postgres.svc.cluster.local - - # Credentials used to initialize fence db if it doesn't exist. - master_user: postgres - master_pass: postgres - - # Actual fence db creds - user: postgres - password: postgres - dbname: arborist \ No newline at end of file diff --git a/helm/argo-wrapper/Chart.yaml b/helm/argo-wrapper/Chart.yaml index 145a49cc..71738137 100644 --- a/helm/argo-wrapper/Chart.yaml +++ b/helm/argo-wrapper/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: argo-wrapper -description: A Helm chart for Argo Wrapper Service +description: A Helm chart for gen3 Argo Wrapper Service # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "master" +appVersion: "2022.10" diff --git a/helm/audit/Chart.lock b/helm/audit/Chart.lock new file mode 100644 index 00000000..5722364c --- /dev/null +++ b/helm/audit/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:9447ea9a4ddee41221215f9d511d904829f457523bc78ddaa817c161e934f27f +generated: "2022-10-20T21:34:34.028897-05:00" diff --git a/helm/audit/Chart.yaml b/helm/audit/Chart.yaml index 4cd2aef5..636f2c95 100644 --- a/helm/audit/Chart.yaml +++ b/helm/audit/Chart.yaml @@ -15,11 +15,17 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 2022.05 +appVersion: "2022.10" + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common + condition: db_creation.enabled diff --git a/helm/audit/templates/_helpers.tpl b/helm/audit/templates/_helpers.tpl index 22c49889..65a9c211 100644 --- a/helm/audit/templates/_helpers.tpl +++ b/helm/audit/templates/_helpers.tpl @@ -79,6 +79,6 @@ Create the name of the service account to use {{- if $localpass }} {{- default (index $localpass.data "postgres-password" | b64dec) }} {{- else }} -{{- default .Values.database.password }} +{{- default .Values.postgres.password }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/audit/templates/db-init.yaml b/helm/audit/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/audit/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/audit/templates/secrets.yaml b/helm/audit/templates/secrets.yaml index 83d21ae9..97e193ec 100644 --- a/helm/audit/templates/secrets.yaml +++ b/helm/audit/templates/secrets.yaml @@ -20,7 +20,7 @@ stringData: #################### # DATABASE # #################### - DB_HOST: {{ .Values.database.host | quote }} - DB_USER: {{ .Values.database.user | quote}} - DB_PASSWORD: {{ include "audit.postgres.password" . | quote }} - DB_DATABASE: {{ .Values.database.dbname | quote }} + DB_HOST: {{ include "gen3.service-postgres" (dict "key" "host" "service" $.Chart.Name "context" $) | quote }} + DB_USER: {{include "gen3.service-postgres" (dict "key" "username" "service" $.Chart.Name "context" $) | quote}} + DB_PASSWORD: {{include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) | quote }} + DB_DATABASE: {{include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $)| quote }} diff --git a/helm/audit/values.yaml b/helm/audit/values.yaml index f195f12a..f9cbec28 100644 --- a/helm/audit/values.yaml +++ b/helm/audit/values.yaml @@ -1,6 +1,26 @@ # Default values for audit. # This is a YAML-formatted file. # Declare variables to be passed into your templates. +global: + # Default values are for postgres deployed as a helm chart + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + +db_create: true +postgres: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: audit + username: audit + host: + port: 5432 + # If left empty password will be auto-generated + password: + + replicaCount: 1 @@ -41,23 +61,6 @@ service: type: ClusterIP port: 80 -ingress: - enabled: true - annotations: { - nginx.ingress.kubernetes.io/rewrite-target: /$2, - kubernetes.io/ingress.class: "nginx" - } - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: ingress.local - paths: - - path: "/audit(/|$)(.*)" - pathType: Prefix - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local resources: requests: @@ -152,17 +155,3 @@ secrets: sqs: url: "http://sqs.com" region: "us-east-1" - - -database: - port: 5432 - host: postgres-postgresql.postgres.svc.cluster.local - - # Credentials used to initialize fence db if it doesn't exist. - master_user: postgres - master_pass: postgres - - # Actual fence db creds - user: postgres - password: postgres - dbname: audit \ No newline at end of file diff --git a/helm/aws-es-proxy/Chart.yaml b/helm/aws-es-proxy/Chart.yaml index 58829a7a..bb8dc186 100644 --- a/helm/aws-es-proxy/Chart.yaml +++ b/helm/aws-es-proxy/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: aws-es-proxy -description: A Helm chart for AWS ES Proxy Service +description: A Helm chart for AWS ES Proxy Service for gen3 # A chart can be either an 'application' or a 'library' chart. # @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/db-setup/.helmignore b/helm/common/.helmignore similarity index 100% rename from helm/db-setup/.helmignore rename to helm/common/.helmignore diff --git a/helm/db-setup/Chart.yaml b/helm/common/Chart.yaml similarity index 90% rename from helm/db-setup/Chart.yaml rename to helm/common/Chart.yaml index 991e18cb..d6f450ed 100644 --- a/helm/db-setup/Chart.yaml +++ b/helm/common/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -name: db-setup -description: A Helm chart for Kubernetes +name: common +description: A Helm chart for provisioning databases in gen3 # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "master" diff --git a/helm/common/templates/_db_setup_job.tpl b/helm/common/templates/_db_setup_job.tpl new file mode 100644 index 00000000..cc45f329 --- /dev/null +++ b/helm/common/templates/_db_setup_job.tpl @@ -0,0 +1,88 @@ +# DB Setup Job +{{- define "common.db-setup-job" -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-dbcreate + annotations: + "helm.sh/hook": "pre-install" #,pre-upgrade" +spec: + template: + metadata: + labels: + app: gen3job + spec: + restartPolicy: OnFailure + containers: + - name: db-setup + image: quay.io/cdis/awshelper:master + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + env: + - name: PGPASSWORD + value: "{{ include "gen3.master-postgres" (dict "key" "password" "context" $) }}" + - name: PGUSER + value: "{{ include "gen3.master-postgres" (dict "key" "username" "context" $) }}" + - name: PGPORT + value: "{{ include "gen3.master-postgres" (dict "key" "port" "context" $) }}" + - name: PGHOST + value: "{{ include "gen3.master-postgres" (dict "key" "host" "context" $) }}" + - name: SERVICE_PGUSER + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: username + optional: false + - name: SERVICE_PGDB + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: database + optional: false + - name: SERVICE_PGPASS + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: password + optional: false + args: + - | + env + echo "SERVICE_PGDB=$SERVICE_PGDB" + echo "SERVICE_PGUSER=$SERVICE_PGUSER" + if psql -lqt | cut -d \| -f 1 | grep -qw $SERVICE_PGDB; then + echo "Database exists" + PGPASSWORD=$SERVICE_PGPASS psql -d $SERVICE_PGDB -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -c "\conninfo" + else + echo "database does not exist" + psql -tc "SELECT 1 FROM pg_database WHERE datname = '$SERVICE_PGDB'" | grep -q 1 || psql -c "CREATE DATABASE $SERVICE_PGDB;" + psql -tc "SELECT 1 FROM pg_user WHERE usename = '$SERVICE_PGUSER'" | grep -q 1 || psql -c "CREATE USER $SERVICE_PGUSER WITH PASSWORD '$SERVICE_PGPASS';" + psql -c "GRANT ALL ON DATABASE $SERVICE_PGDB TO $SERVICE_PGUSER WITH GRANT OPTION;" + psql -d $SERVICE_PGDB -c "CREATE EXTENSION ltree; ALTER ROLE $SERVICE_PGUSER WITH LOGIN" + PGPASSWORD=$SERVICE_PGPASS psql -d $SERVICE_PGDB -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -c "\conninfo" + fi +{{- end }} + + +{{/* +Create k8s secrets for connecting to postgres +*/}} +# DB Secrets +{{- define "common.db-secret" -}} +{{- if not (lookup "v1" "Secret" .Release.Namespace (printf "%s-%s" .Chart.Name "dbcreds")) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $.Chart.Name }}-dbcreds + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/resource-policy": "keep" + "helm.sh/hook-weight": "-10" +stringData: + host: {{ include "gen3.service-postgres" (dict "key" "host" "service" $.Chart.Name "context" $) }} + database: "{{ include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $) }}" + username: "{{ include "gen3.service-postgres" (dict "key" "username" "service" $.Chart.Name "context" $) }}" + password: "{{ include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }}" + port: "{{ include "gen3.service-postgres" (dict "key" "port" "service" $.Chart.Name "context" $) }}" +{{- end -}} +{{- end }} \ No newline at end of file diff --git a/helm/common/templates/_postgres_secrets.tpl b/helm/common/templates/_postgres_secrets.tpl new file mode 100644 index 00000000..4cf5c8f1 --- /dev/null +++ b/helm/common/templates/_postgres_secrets.tpl @@ -0,0 +1,94 @@ +{{/* + Postgres service secret lookup. + Usage: + {{ include "gen3.service-postgres" (dict "key" "password" "service" "fence" "context" $) }} + + + Params: + - key - String - Required - Name of the key in the secret. + - service - String - Which service are you looking up secret for? + - context - Context - Required - Parent context. + + + Lookups for postgres service secret is done in this order, until it finds a value: + - Secret provided via `.Values.postgres` (Can be database, username, password, host, port) + - Lookup secret `{{service}}-dbcreds` with key `password` + - Generate a random string, as we can assume this is a fresh install at that point. + +*/}} +{{- define "gen3.service-postgres" -}} + {{- $chartName := default "" .context.Chart.Name }} + {{- $valuesPostgres := get .context.Values.postgres .key}} + {{- $localSecretPass := get ((lookup "v1" "Secret" .context.Release.Namespace (cat .service "-dbcreds")).data) .key }} + + {{- $randomPassword := "" }} + {{- $valuesGlobalPostgres := get .context.Values.global.postgres.master .key}} + {{- if eq .key "password" }} + {{- $randomPassword = randAlphaNum 20 }} + {{- $valuesGlobalPostgres = "" }} + {{- end }} + {{- $password := coalesce $valuesPostgres $localSecretPass $randomPassword $valuesGlobalPostgres}} + {{- printf "%v" $password -}} +{{- end }} + + +{{/* +Postgres Master Secret Lookup + +Usage: + {{ include "gen3.master-postgres" (dict "key" "database" "context" $) }} + + Lookups for secret is done in this order, until it finds a value: + - Secret provided via `.Values.global.master.postgres` (Can be database, username, password, host, port) + - Lookup secret `postgres-postgresql` with property `postgres-password` in `postgres` namespace. (This is for develop installation of gen3) + + + # https://helm.sh/docs/chart_template_guide/function_list/#coalesce +*/}} +{{- define "gen3.master-postgres" }} + {{- $chartName := default "" .context.Chart.Name }} + + {{- $valuesPostgres := get .context.Values.global.postgres.master .key}} + {{- $secret := (lookup "v1" "Secret" "postgres" "postgres-postgresql" )}} + {{- $devPostgresSecret := "" }} + {{- if $secret }} + {{- $devPostgresSecret = (index $secret.data "postgres-password") | b64dec }} + {{- end }} + {{- $value := coalesce $valuesPostgres $devPostgresSecret }} + {{- printf "%v" $value -}} +{{- end }} + + + + + + +{{/* + Postgres User lookup +*/}} +{{- define "peregrine.postgres.user" -}} +{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} +{{- if $localpass }} +{{- default (index $localpass.data "postgres-password" | b64dec) }} +{{- else }} +{{- default .Values.postgres.password }} +{{- end }} +{{- end }} + + +{{- if not (lookup "v1" "Secret" .Release.Namespace (printf "%s-%s" .Chart.Name "dbcreds")) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Chart.Name }}-dbcreds + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/resource-policy": "keep" + "helm.sh/hook-weight": "-10" +stringData: + host: {{ default .Values.global.postgres.host .Values.postgres.host }} + database: "{{ default .Chart.Name .Values.postgres.dbname }}" + username: "{{ default .Chart.Name .Values.postgres.user }}" + password: "{{ default (randAlphaNum 24 | nospace) .Values.postgres.password }}" + port: "{{ default 5432 .Values.postgres.port }}" +{{- end -}} \ No newline at end of file diff --git a/helm/common/templates/_secrets.tpl b/helm/common/templates/_secrets.tpl new file mode 100644 index 00000000..837aa104 --- /dev/null +++ b/helm/common/templates/_secrets.tpl @@ -0,0 +1,140 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" $.context.Release.Namespace .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key | quote }} + {{- else }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/helm/db-setup/templates/_db_setup.tpl b/helm/db-setup/templates/_db_setup.tpl deleted file mode 100644 index 8b187f1c..00000000 --- a/helm/db-setup/templates/_db_setup.tpl +++ /dev/null @@ -1,31 +0,0 @@ -{{- define "db-setup.setup-job" -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: db-setup -spec: - template: - metadata: - labels: - app: gen3job - annotations: - "helm.sh/hook": pre-install - "helm.sh/hook-weight": "-5" - "helm.sh/hook-delete-policy": hook-succeeded - spec: - restartPolicy: OnFailure - containers: - - name: db-setup - image: quay.io/cdis/awshelper:master - imagePullPolicy: Always - command: ["/bin/bash", "-c"] - args: - - | - export PGPASSWORD="{{ $.Values.postgres.master.password }}" - {{- range .Values.postgres.databases }} - psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -tc "SELECT 1 FROM pg_database WHERE datname = '{{ .databaseName }}'" | grep -q 1 || psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -c "CREATE DATABASE {{ .databaseName }};" - psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -tc "SELECT 1 FROM pg_user WHERE usename = '{{ .username }}'" | grep -q 1 || psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -c "CREATE USER {{ .username }} WITH PASSWORD '{{ .password }}';" - psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -c "GRANT ALL ON DATABASE {{ .databaseName }} TO {{ .username }} WITH GRANT OPTION;" - psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -d {{ .databaseName }} -c "CREATE EXTENSION ltree; ALTER ROLE {{ .username }} WITH LOGIN" - {{- end }} -{{- end -}} \ No newline at end of file diff --git a/helm/db-setup/values.yaml b/helm/db-setup/values.yaml deleted file mode 100644 index a2d8c7b3..00000000 --- a/helm/db-setup/values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -postgres: - host: postgres-postgresql.postgres.svc.cluster.local - - master: - username: postgres - password: postgres - port: 5432 - - databases: - - databaseName: wts - username: wts - password: wts_password - - databaseName: indexd - username: indexd - password: indexd_password \ No newline at end of file diff --git a/helm/dicom-server/Chart.yaml b/helm/dicom-server/Chart.yaml index e14931c4..f5014981 100644 --- a/helm/dicom-server/Chart.yaml +++ b/helm/dicom-server/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: dicom-server -description: A Helm chart for AWS Dicom Server +description: A Helm chart for gen3 Dicom Server # A chart can be either an 'application' or a 'library' chart. # diff --git a/helm/dicom-viewer/Chart.yaml b/helm/dicom-viewer/Chart.yaml index 020fcdde..7f8012b9 100644 --- a/helm/dicom-viewer/Chart.yaml +++ b/helm/dicom-viewer/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: dicom-viewer -description: A Helm chart for AWS Dicom Viewer +description: A Helm chart for gen3 Dicom Viewer # A chart can be either an 'application' or a 'library' chart. # diff --git a/helm/fence/Chart.lock b/helm/fence/Chart.lock new file mode 100644 index 00000000..0d7b33f1 --- /dev/null +++ b/helm/fence/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:9447ea9a4ddee41221215f9d511d904829f457523bc78ddaa817c161e934f27f +generated: "2022-10-20T21:34:35.742578-05:00" diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index ca9b5879..75e40506 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: fence -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 Fence # A chart can be either an 'application' or a 'library' chart. # @@ -15,9 +15,15 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 2022.05 \ No newline at end of file +appVersion: "2022.10" + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common + condition: db_creation.enabled \ No newline at end of file diff --git a/helm/fence/fence-config/fence-config.yaml b/helm/fence/fence-config/fence-config.yaml deleted file mode 100644 index db24752e..00000000 --- a/helm/fence/fence-config/fence-config.yaml +++ /dev/null @@ -1,912 +0,0 @@ ---- -############################### Fence Configuration #################################### -# This file contains various configurations for the Fence microservice. -# -# README: -# - This is initially configured for minimal local development with reasonable defaults. -# - Descriptions for each of the configurations (if any) will be *above* the variable as -# comments. -# - Some configuration variables will have examples commented out below them. -# - This is broken up into 2 main sections for REQUIRED and OPTIONAL configurations. -# - Optional configs will note what features or endpoints they support -# - Underneath each main section the variables are logically grouped under named -# sections. -# -# NOTE: Login is NOT ready out of the box. Fill out REQUIRED configurations first - -######################################################################################## -# REQUIRED CONFIGURATIONS # -######################################################################################## - -# ////////////////////////////////////////////////////////////////////////////////////// -# GENERAL -# - Fill out all variables! -# ////////////////////////////////////////////////////////////////////////////////////// -APP_NAME: 'Gen3 Data Commons' -# Where fence microservice is deployed -BASE_URL: 'http://localhost/user' -# postgres db to connect to -# connection url format: -# postgresql://[user[:password]@][netloc][:port][/dbname] -DB: 'postgresql://postgres:password!123@fence-postgresql:5432/fence' - -# A URL-safe base64-encoded 32-byte key for encrypting keys in db -# in python you can use the following script to generate one: -# import base64 -# import os -# key = base64.urlsafe_b64encode(os.urandom(32)) -# print(key) -ENCRYPTION_KEY: '' - -# ////////////////////////////////////////////////////////////////////////////////////// -# DEBUG & SECURITY SETTINGS -# - Modify based on whether you're in a dev environment or in production -# ////////////////////////////////////////////////////////////////////////////////////// -# flask's debug setting -# WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) -DEBUG: true -# if true, will automatically login a user with username "test" -# WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) -MOCK_AUTH: false -# if true, will fake a successful login response from Google in /login/google -# NOTE: this will also modify the behavior of /link/google endpoints -# WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) -# will login as the username set in cookie DEV_LOGIN_COOKIE_NAME -MOCK_GOOGLE_AUTH: false -DEV_LOGIN_COOKIE_NAME: "dev_login" -# if true, will ignore anything configured in STORAGE_CREDENTIALS -MOCK_STORAGE: true -# allow OIDC traffic on http for development. By default it requires https. -# -# WARNING: ONLY set to true when fence will be deployed in such a way that it will -# ONLY receive traffic from internal clients and can safely use HTTP. -AUTHLIB_INSECURE_TRANSPORT: true -# enable Prometheus Metrics for observability purposes -# -# WARNING: Any counters, gauges, histograms, etc. should be carefully -# reviewed to make sure its labels do not contain any PII / PHI -ENABLE_PROMETHEUS_METRICS: false - -# set if you want browsers to only send cookies with requests over HTTPS -SESSION_COOKIE_SECURE: true - -ENABLE_CSRF_PROTECTION: true - -# Signing key for WTForms to sign CSRF tokens with -WTF_CSRF_SECRET_KEY: '{{ENCRYPTION_KEY}}' - -# fence (at the moment) attempts a migration on startup. setting this to false will disable that -# WARNING: ONLY set to false if you do NOT want to automatically migrate your database. -# You should be careful about incompatible versions of your db schema with what -# fence expects. In other words, things could be broken if you update to a later -# fence that expects a schema your database isn't migrated to. -# NOTE: We are working to improve the migration process in the near future -ENABLE_DB_MIGRATION: true - -# ////////////////////////////////////////////////////////////////////////////////////// -# OPEN ID CONNECT (OIDC) -# - Fully configure at least one client so login works -# - WARNING: Be careful changing the *_ALLOWED_SCOPES as you can break basic -# and optional functionality -# ////////////////////////////////////////////////////////////////////////////////////// -OPENID_CONNECT: - # any OIDC IDP that does not differ from the generic implementation can be - # configured without code changes - generic_oidc_idp: # choose a unique ID and replace this key - name: 'some_idp' # optional; display name for this IDP - client_id: '' - client_secret: '' - redirect_url: '{{BASE_URL}}/login/some_idp/login' # replace IDP name - # use `discovery` to configure IDPs that do not expose a discovery - # endpoint. One of `discovery_url` or `discovery` should be configured - discovery_url: 'https://server.com/.well-known/openid-configuration' - discovery: - authorization_endpoint: '' - token_endpoint: '' - jwks_uri: '' - user_id_field: '' # optional (default "sub"); claims field to get the user_id from - email_field: '' # optional (default "email"); claims field to get the user email from - scope: '' # optional (default "openid") - # These Google values must be obtained from Google's Cloud Console - # Follow: https://developers.google.com/identity/protocols/OpenIDConnect - # - # You'll need to obtain a Client ID and Client Secret. Set the redirect URIs - # in Google to be '{{BASE_URL}}/login/google/login', but expand BASE_URL to - # whatever you set it to above. - google: - discovery_url: 'https://accounts.google.com/.well-known/openid-configuration' - client_id: '' - client_secret: '' - # this is be the allowed redirect back to fence, should not need to change - redirect_url: '{{BASE_URL}}/login/google/login/' - scope: 'openid email' - # if mock is true, will fake a successful login response from Google in /login/google - # NOTE: this will also modify the behavior of /link/google endpoints - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - # will login as the username set in cookie DEV_LOGIN_COOKIE_NAME or default provided - # here - mock: '{{MOCK_GOOGLE_AUTH}}' # for backwards compatibility with older cfg files - mock_default_user: 'test@example.com' - # Support for multi-tenant fence (another fence is this fence's IDP) - # If this fence instance is a client of another fence, fill this cfg out. - # REMOVE if not needed - fence: - # this api_base_url should be the root url for the OTHER fence - # something like: https://example.com - api_base_url: '' - # this client_id and client_secret should be obtained by registering THIS fence as - # a new client of the OTHER fence - client_id: '' - client_secret: '' - client_kwargs: - # openid is required to use OIDC flow - scope: 'openid' - # callback after logging in through the other fence - redirect_uri: '{{BASE_URL}}/login/fence/login' - # The next 3 should not need to be changed if the provider is following - # Oauth2 endpoint naming conventions - authorize_url: '{{api_base_url}}/oauth2/authorize' - access_token_url: '{{api_base_url}}/oauth2/token' - refresh_token_url: '{{api_base_url}}/oauth2/token' - # Custom name to display for consent screens. If not provided, will use `fence`. - # If the other fence is using NIH Login, you should make name: `NIH Login` - name: '' - # if mock is true, will fake a successful login response for login - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - mock: false - mock_default_user: 'test@example.com' - # this is needed to enable InCommon login, if some LOGIN_OPTIONS are configured with idp=fence and a list of shib_idps: - shibboleth_discovery_url: 'https://login.bionimbus.org/Shibboleth.sso/DiscoFeed' - # you can setup up an orcid client here: https://orcid.org/developer-tools - orcid: - discovery_url: 'https://orcid.org/.well-known/openid-configuration' - client_id: '' - client_secret: '' - # make sure you put the FULL url for this deployment in the allowed redirects in - # ORCID.org. DO NOT include {{BASE_URL}} at ORCID.org, you need to actually put the - # full url - redirect_url: '{{BASE_URL}}/login/orcid/login/' - scope: 'openid' - # if mock is true, will fake a successful login response for login - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - mock: false - mock_default_user: '0000-0002-2601-8132' - ras: - discovery_url: 'https://sts.nih.gov/.well-known/openid-configuration' - client_id: '' - client_secret: '' - redirect_url: '{{BASE_URL}}/login/ras/callback' - scope: 'openid email profile ga4gh_passport_v1' - # if mock is true, will fake a successful login response for login - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - mock: false - mock_default_user: 'test@example.com' - # Create a client in Azure here: - # https://portal.azure.com/#blade/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/RegisteredAppsPreview - # Currently supports organizational account only, so when registering a new App in - # Azure, make sure to select the `Accounts in any organizational directory` for - # supported account types. - microsoft: - discovery_url: 'https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration' - # after registering a new appl, client_id can be found as - # "APPLICATION (CLIENT) ID" in Microsoft Azure - client_id: '' - # You have a generate a secret in Azure for this app, there should be a - # "Certificates & secrets" section where you can create a "New client secret" - client_secret: '' - # make sure you put the FULL url for this deployment in the allowed redirects in - # your app in Azure. DO NOT include {{BASE_URL}} in Azure, you need to actually put the - # full url - redirect_url: '{{BASE_URL}}/login/microsoft/login/' - scope: 'openid email' - # if mock is true, will fake a successful login response for login - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - mock: false - mock_default_user: 'test@example.com' - # For information on configuring an Okta tenant as an OIDC IdP refer to Okta documentation at: - # https://developer.okta.com/docs/reference/api/oidc/#2-okta-as-the-identity-platform-for-your-app-or-api - okta: - discovery_url: '' - client_id: '' - client_secret: '' - redirect_url: '{{BASE_URL}}/login/okta/login/' - scope: 'openid email' - cognito: - # You must create a user pool in order to have a discovery url - discovery_url: 'https://cognito-idp.{REGION}.amazonaws.com/{USER-POOL-ID}/.well-known/openid-configuration' - client_id: '' - client_secret: '' - redirect_url: '{{BASE_URL}}/login/cognito/login/' - scope: 'openid email' - # In the case where Cognito is being used solely as an intermediary to a single IdP, - # and that IdP is a SAML IdP with no 'email_verified' outgoing claim, but it is safe - # to assume all emails from this SAML IdP are in fact verified, we may set this to True - assume_emails_verified: False - # CILogon subscribers can create and manage OIDC clients using COmanage Registry. - # Free tier users may request OIDC clients at https://cilogon.org/oauth2/register - cilogon: - discovery_url: 'https://cilogon.org/.well-known/openid-configuration' - client_id: '' - client_secret: '' - # When registering the Callback URLs for your CILogon OIDC client be - # sure to include the FULL url for this deployment, including the https:// scheme - # and server FQDN. - redirect_url: '{{BASE_URL}}/login/cilogon/login/' - scope: 'openid email profile' - # if mock is true, will fake a successful login response for login - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - mock: false - mock_default_user: 'http://cilogon.org/serverT/users/64703' - synapse: - discovery_url: '' - client_id: '' - client_secret: '' - redirect_url: '' - scope: 'openid' - shibboleth: - client_id: '' - client_secret: '' - redirect_url: '{{BASE_URL}}/login/shib/login' - -# these are the *possible* scopes a client can be given, NOT scopes that are -# given to all clients. You can be more restrictive during client creation -CLIENT_ALLOWED_SCOPES: - - "openid" - - "user" - - "data" - - "google_credentials" - - "google_service_account" - - "google_link" - - "ga4gh_passport_v1" - -# these are the scopes that CAN be included in a user's own access_token -USER_ALLOWED_SCOPES: - - "fence" - - "openid" - - "user" - - "data" - - "admin" - - "google_credentials" - - "google_service_account" - - "google_link" - - "ga4gh_passport_v1" - -# these are the scopes that a browser session can create for a user (very -# similar to USER_ALLOWED_SCOPES, as the session will actually create access_tokens -# for an actively logged in user) -SESSION_ALLOWED_SCOPES: - - "openid" - - "user" - - "credentials" - - "data" - - "admin" - - "google_credentials" - - "google_service_account" - - "google_link" - - "ga4gh_passport_v1" - -# ////////////////////////////////////////////////////////////////////////////////////// -# LOGIN -# - Modify based on which OIDC provider(s) you configured above -# - NOTE: You can have multiple IDPs for users to login with, but one has to be set -# as the default -# ////////////////////////////////////////////////////////////////////////////////////// - -# List of enabled login options (used by data-portal to display login buttons). -# Each option must be configured with a "name" and an "idp". -# - "idp" must be a configured provider in OPENID_CONNECT section. -# Multiple options can be configured with the same idp. -# - if provider_id is "fence", "fence_idp" can be any of the providers -# supported by the other Fence. If not specified, will default to NIH login. -# - if provider_id is "fence" and fence_idp is "shibboleth", a list of -# "shib_idps" can be configured for InCommon login. If not specified, will -# default to NIH login. -# - Optional parameters: "desc" (description) and "secondary" (boolean - can -# be used by the frontend to display secondary buttons differently). -LOGIN_OPTIONS: [] # !!! remove the empty list to enable login options! - # - name: 'Login from Google' - # desc: 'description' - # idp: google - # secondary: True - # - name: 'ORCID Login' - # idp: orcid - # - name: 'Microsoft Login' - # idp: microsoft - # - name: 'Okta Login' - # idp: okta - # # Cognito login: You may want to edit the name to reflect Cognito's IdP, - # # especially if Cognito is only using one IdP - # - name: 'Login from Cognito' - # desc: 'Amazon Cognito login' - # idp: cognito - # - name: 'Login from RAS' - # idp: ras - # - name: 'NIH Login' - # idp: fence - # fence_idp: shibboleth - # - name: 'ORCID Login through other Fence' - # idp: fence - # fence_idp: orcid - # - name: 'CILogon Login' - # idp: cilogon - # - name: 'InCommon Login' - # idp: fence - # fence_idp: shibboleth - # # "shib_idps" can be '*' or a list of one or more entity IDs - # shib_idps: - # - urn:mace:incommon:nih.gov - # - urn:mace:incommon:uchicago.edu -# The following can be used for shibboleth login, simply uncomment. -# NOTE: Don't enable shibboleth if the deployment is not protected by -# shibboleth module, the shib module takes care of preventing header -# spoofing. - # - name: 'Shibboleth Login' - # idp: shibboleth - -# Default login provider: -# - must be configured in LOGIN_OPTIONS and OPENID_CONNECT -# - if several options in LOGIN_OPTIONS are defined for this IDP, will default -# to the first one. -DEFAULT_LOGIN_IDP: null - -# Default login URL: DEPRECATED and replaced by LOGIN_OPTIONS + DEFAULT_LOGIN_IDP configs -# - Google? Use: '{{BASE_URL}}/login/google' -# - Multi-tenant fence (e.g. another fence instance)? Use: '{{BASE_URL}}/login/fence' -# - Sibboleth? Use: '{{BASE_URL}}/login/shib' -DEFAULT_LOGIN_URL: '{{BASE_URL}}/login/google' - -# `LOGIN_REDIRECT_WHITELIST` is a list of extra whitelisted URLs which can be redirected -# to by the `/login/*` endpoints. Fence automatically populates this with the redirect -# URLs for any registered OAuth clients, and its own URL. When validating the redirects, -# fence chesk whether the domain for the redirect matches a domain in the whitelist (so -# only the domains for the additional desired redirects are necessary here). -LOGIN_REDIRECT_WHITELIST: [] - -### DEPRECATED and replaced by OPENID_CONNECT + LOGIN_OPTIONS configs -ENABLED_IDENTITY_PROVIDERS: {} - - -# ////////////////////////////////////////////////////////////////////////////////////// -# LIBRARY CONFIGURATION (authlib & flask) -# - Already contains reasonable defaults -# ////////////////////////////////////////////////////////////////////////////////////// -# authlib-specific configs for OIDC flow and JWTs -# NOTE: the OAUTH2_JWT_KEY cfg gets set automatically by fence if keys are setup -# correctly -OAUTH2_JWT_ALG: 'RS256' -OAUTH2_JWT_ENABLED: true -OAUTH2_JWT_ISS: '{{BASE_URL}}' -OAUTH2_PROVIDER_ERROR_URI: '/api/oauth2/errors' - -# used for flask, "path mounted under by the application / web server" -# since we deploy as microservices, fence is typically under {{base}}/user -# this is also why our BASE_URL default ends in /user -APPLICATION_ROOT: '/user' - - -# ////////////////////////////////////////////////////////////////////////////////////// -# Tokens, Lifetimes, & Expirations -# - Already contains reasonable defaults -# ////////////////////////////////////////////////////////////////////////////////////// -# The name of the browser cookie in which the access token will be stored. -ACCESS_TOKEN_COOKIE_NAME: "access_token" - -# The name of the browser cookie in which the session token will be stored. -# Note that the session token also stores information for the -# ``flask.session`` in the ``context`` field of the token. -SESSION_COOKIE_NAME: "fence" - -# The domain of the browser cookie in which the session token will be stored. -# Leave unset (not empty string!) for normal single-site deployment. -SESSION_COOKIE_DOMAIN: - -OAUTH2_TOKEN_EXPIRES_IN: - "authorization_code": 1200 - "implicit": 1200 - -# The number of seconds after an access token is issued until it expires. -ACCESS_TOKEN_EXPIRES_IN: 1200 - -# The number of seconds after a refresh token is issued until it expires. -REFRESH_TOKEN_EXPIRES_IN: 2592000 - -# The number of seconds after which a browser session is considered stale. -SESSION_TIMEOUT: 1800 - -# The maximum session lifetime in seconds. -SESSION_LIFETIME: 28800 - -# The number of seconds the user's Google service account key used for -# url signing will last before being expired/rotated -# 30 days: 2592000 seconds -GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN: 2592000 - -# The number of seconds after a User's Google Service account is added to bucket -# access until it expires. -# 7 days: 604800 seconds -GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN: 604800 - -# The number of seconds after a User's Google account is added to bucket -# access until it expires. -GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN: 86400 - -# The number of seconds after a pre-signed url is issued until it expires. -MAX_PRESIGNED_URL_TTL: 3600 - -# The number of seconds after an API KEY is issued until it expires. -MAX_API_KEY_TTL: 2592000 - -# The number of seconds after an access token is issued until it expires. -MAX_ACCESS_TOKEN_TTL: 3600 - -# TEMPORARY: The maximum number of projects allowed in token claims. -# This config var should be removed after sheepdog and peregrine support -# auth checks against Arborist, and no longer check the token. -TOKEN_PROJECTS_CUTOFF: 10 - -# If set to true, will generate an new access token each time when a browser session update happens -RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION: false - -# The maximum lifetime of a Gen3 passport in seconds -GEN3_PASSPORT_EXPIRES_IN: 43200 - -######################################################################################## -# OPTIONAL CONFIGURATIONS # -######################################################################################## - -# For displaying a privacy policy to users, we can either link to the URL specified by -# PRIVACY_POLICY_URL, or default to the `static/privacy_policy.md` file in fence. -PRIVACY_POLICY_URL: null - -# ////////////////////////////////////////////////////////////////////////////////////// -# RELIABILITY OPTS -# ////////////////////////////////////////////////////////////////////////////////////// -# Configurations related to resiliency, fault-tolerance and availability -# This is the number of requests per second that the Nginx proxy will accept before reaching fence -# The value defined in fence-config-public.yaml takes precedence over this one -# In the absence of this OVERRIDE prefixed config, the legacy NGINX_RATE_LIMIT from the k8s deployment yaml is applied -OVERRIDE_NGINX_RATE_LIMIT: 18 - -# ////////////////////////////////////////////////////////////////////////////////////// -# SUPPORT INFO -# ////////////////////////////////////////////////////////////////////////////////////// -# If you want an email address to show up when an unhandled error occurs, provide one -# here. Something like: support@example.com -SUPPORT_EMAIL_FOR_ERRORS: null - -# ////////////////////////////////////////////////////////////////////////////////////// -# SHIBBOLETH -# - Support using `shibboleth` in LOGIN_OPTIONS -# - Contains defaults for using NIH's Login. -# ////////////////////////////////////////////////////////////////////////////////////// -# assumes shibboleth is deployed under {{BASE_URL}}/shibboleth -SHIBBOLETH_HEADER: 'persistent_id' -SSO_URL: 'https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=' -ITRUST_GLOBAL_LOGOUT: 'https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=' - -# ////////////////////////////////////////////////////////////////////////////////////// -# dbGaP USER SYNCING SUPPORT -# - Support syncing authorization information from dbGaP -# ////////////////////////////////////////////////////////////////////////////////////// -# "dbGaP project serves as an access gateway for researchers seeking to gain -# access to genotype and phenotype data" -# -# User syncing and access can also be done throught a User Access file. See -# fence's README for more information -dbGaP: - - info: - host: '' - username: '' - password: '' - port: 22 - proxy: '' - proxy_user: '' - protocol: 'sftp' - decrypt_key: '' - # parse out the consent from the dbgap accession number such that something - # like "phs000123.v1.p1.c2" becomes "phs000123.c2". - # - # NOTE: when this is "false" the above would become "phs000123" - parse_consent_code: true - # A consent of "c999" can indicate access to that study's "exchange area data" - # and when a user has access to one study's exchange area data, they - # have access to the parent study's "common exchange area data" that is not study - # specific. The following config is whether or not to parse/handle "c999" codes - # for access to the common exchange area data - # - # NOTE: When enabled you MUST also provide a mapping to the - # `study_common_exchange_areas` from study -> parent common exchange area resource - enable_common_exchange_area_access: false - # The below configuration is a mapping from studies to their "common exchange area data" - # Fence project name a user gets access to when parsing c999 exchange area codes (and - # subsequently gives access to an Arborist resource representing this common area - # as well) - study_common_exchange_areas: - 'example': 'test_common_exchange_area' - # 'studyX': 'test_common_exchange_area' - # 'studyY': 'test_common_exchange_area' - # 'studyZ': 'test_common_exchange_area' - # A mapping from the dbgap study / Fence project to which authorization namespaces the - # actual data lives in. For example, `studyX` data may exist in multiple organizations, so - # we need to know how to map authorization to all orgs resources - study_to_resource_namespaces: - '_default': ['/'] - 'test_common_exchange_area': ['/dbgap/'] - # above are for default support and exchange area support - # below are further examples - # - # 'studyX': ['/orgA/', '/orgB/'] - # 'studyX.c2': ['/orgB/', '/orgC/'] - # 'studyZ': ['/orgD/'] -# Regex to match an assession number that has consent information in forms like: -# phs00301123.c999 -# phs000123.v3.p1.c3 -# phs000123.c3 -# phs00301123.v3.p4.c999 -# Will NOT MATCH forms like: phs000123 -# -# WARNING: Do not change this without consulting the code that uses it -DBGAP_ACCESSION_WITH_CONSENT_REGEX: '(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)' - -# ////////////////////////////////////////////////////////////////////////////////////// -# STORAGE BACKENDS AND CREDENTIALS -# - Optional: Used for `/admin` & `/credentials` endpoints for user management. -# Also used during User Syncing process to automate managing Storage -# access for users. -# ////////////////////////////////////////////////////////////////////////////////////// -# When true, this modifies usersync (not fence service itself) such that when syncing user -# access to a Google storage backend happens in "bulk" by doing a diff *per google group* -# between what's in Google and what's expected. Then it adds, removes only as necessary. -# This is in contrast to the default logic which does blind updates per user and ignores -# 409s from Google. -# NOTE: This reduces the number of API calls to Google in the general case, but increases -# memory usages by usersync (as it has to track all the Google groups and user access) -GOOGLE_BULK_UPDATES: false - -# Configuration for various storage systems for the backend -# NOTE: Remove the {} and supply backends if needed. Example in comments below -STORAGE_CREDENTIALS: {} -# Google Cloud Storage backend -# -# 'google': -# backend: 'google' -# # this should be the project id where the Google Groups for data access are managed -# google_project_id: 'some-project-id-12378923' - -# Cleversafe data storage backend -# -# 'cleversafe-server-a': -# backend: 'cleversafe' -# aws_access_key_id: '' -# aws_secret_access_key: '' -# host: 'somemanager.osdc.io' -# public_host: 'someobjstore.example.com' -# port: 443 -# is_secure: true -# username: 'someone' -# password: 'somepass' -# is_mocked: true - -# ////////////////////////////////////////////////////////////////////////////////////// -# AWS BUCKETS AND CREDENTIALS -# - Support `/data` endpoints -# ////////////////////////////////////////////////////////////////////////////////////// -AWS_CREDENTIALS: {} -# NOTE: Remove the {} and supply creds if needed. Example in comments below -# 'CRED1': -# aws_access_key_id: '' -# aws_secret_access_key: '' -# 'CRED2': -# aws_access_key_id: '' -# aws_secret_access_key: '' - -# NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a -# call to GetBucketLocation which you make lack the AWS ACLs for. -# public buckets do not need the region field. -# the cred values should be keys in section `AWS_CREDENTIALS`. -S3_BUCKETS: {} -# NOTE: Remove the {} and supply buckets if needed. Example in comments below -# bucket1: -# cred: 'CRED1' -# region: 'us-east-1' -# # optionally you can manually specify an s3-compliant endpoint for this bucket -# endpoint_url: 'https://cleversafe.example.com/' -# bucket2: -# cred: 'CRED2' -# region: 'us-east-1' -# bucket3: -# cred: '*' # public bucket -# bucket4: -# cred: 'CRED1' -# region: 'us-east-1' -# role-arn: 'arn:aws:iam::role1' - -# `DATA_UPLOAD_BUCKET` specifies an S3 bucket to which data files are uploaded, -# using the `/data/upload` endpoint. This must be one of the first keys under -# `S3_BUCKETS` (since these are the buckets fence has credentials for). -DATA_UPLOAD_BUCKET: 'bucket1' - -# ////////////////////////////////////////////////////////////////////////////////////// -# PROXY -# - Optional: If the api is behind firewall that needs to set http proxy -# ////////////////////////////////////////////////////////////////////////////////////// -# NOTE: leave as-is to not use proxy -# this is only used by the Google Oauth2Client at the moment if provided -HTTP_PROXY: - host: null - port: 3128 - -# ////////////////////////////////////////////////////////////////////////////////////// -# MICROSERVICE PATHS -# - Support `/data` endpoints & authz functionality -# ////////////////////////////////////////////////////////////////////////////////////// -# url where indexd microservice is running (for signed urls primarily) -# NOTE: Leaving as null will force fence to default to {{BASE_URL}}/index -# example value: 'https://example.com/index' -INDEXD: null - -# this is the username which fence uses to make authenticated requests to indexd -INDEXD_USERNAME: 'fence' -# this is the password which fence uses to make authenticated requests to indexd -INDEXD_PASSWORD: '' - -# ////////////////////////////////////////////////////////////////////////////////////// -# AZURE STORAGE BLOB CONFIGURATION -# - Support Azure Blob Data Access Methods -# ////////////////////////////////////////////////////////////////////////////////////// - -# https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&tabs=azure-portal#view-account-access-keys -# AZ_BLOB_CREDENTIALS: 'fake connection string' -AZ_BLOB_CREDENTIALS: - -# AZ_BLOB_CONTAINER_URL: 'https://storageaccount.blob.core.windows.net/container/' -# this is the container used for uploading, and should match the storage account -# used in the connection string for AZ_BLOB_CREDENTIALS -AZ_BLOB_CONTAINER_URL: 'https://myfakeblob.blob.core.windows.net/my-fake-container/' - -# url where authz microservice is running -ARBORIST: null - -# url where the audit-service is running -AUDIT_SERVICE: 'http://audit-service' -ENABLE_AUDIT_LOGS: - presigned_url: false - login: false -# `PUSH_AUDIT_LOGS_CONFIG.type` is one of: [api, aws_sqs]. -# - if type == api: logs are created by hitting the log creation endpoint. -# - if type == aws_sqs: logs are pushed to an SQS and `aws_sqs_config` fields -# `sqs_url` and `region` are required. Field `aws_cred` is optional and it -# should be a key in section `AWS_CREDENTIALS`. -PUSH_AUDIT_LOGS_CONFIG: - type: aws_sqs - aws_sqs_config: - sqs_url: - region: - aws_cred: - -# ////////////////////////////////////////////////////////////////////////////////////// -# CLOUD API LIBRARY (CIRRUS) AND GOOGLE CONFIGURATION -# - Support Google Data Access Methods -# ////////////////////////////////////////////////////////////////////////////////////// -# Setting this up allows fence to create buckets, manage Google groups, etc. -# See directions here for setting up cirrus: https://github.com/uc-cdis/cirrus -CIRRUS_CFG: - GOOGLE_API_KEY: '' - GOOGLE_PROJECT_ID: '' - GOOGLE_APPLICATION_CREDENTIALS: '' - GOOGLE_STORAGE_CREDS: '' - GOOGLE_ADMIN_EMAIL: '' - GOOGLE_IDENTITY_DOMAIN: '' - GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL: '' - -# Prefix to namespace Google Groups on a single Cloud Identity (see cirrus -# setup for more info on Cloud Identity) -# -# NOTE: Make this short! Less than 8 characters if possible. Google has -# length restrictions on group names. -GOOGLE_GROUP_PREFIX: '' - -# Prefix to namespace Google Service Accounts in a single Google Cloud Platform Project. -# This is primarily to support multiple instances of fence references the same Google -# project. If that is not something you need to support, then you can leave this blank. -# -# NOTE: Make this short! Less than 8 characters if possible. Google has -# length restrictions on service account names. -GOOGLE_SERVICE_ACCOUNT_PREFIX: '' - -# A Google Project identitifier representing the default project to bill to for -# accessing Google Requester Pays buckets (for signed urls and/or temporary service account -# credentials). If this is provided and the API call for -# Google access does not include a `userProject`, this will be used instead. -# -# WARNING: Setting this WITHOUT setting "ENABLE_AUTOMATIC_BILLING_*" to `true` below, -# means that clients and end-users will be responsible for making sure that -# the service account used in either of these methods actually has billing -# permission in the specified project. -BILLING_PROJECT_FOR_SIGNED_URLS: -BILLING_PROJECT_FOR_SA_CREDS: - -# Setting this to `true` will make Fence automatically attempt to create a Custom Role -# in the billing project and give the necessary Google Service Account that role -# (which will allow it to bill to the project). -# -# NOTE: The Fence SA will need the necessary permissions in the specified project to -# both create a custom role and update the Project's IAM Policy to include the -# necessary SA. At the time of writing, there are pre-defined roles in Google's -# IAM that provide the necessary permissions. Those are "Project IAM Admin" and -# "Role Administrator" -# -# NOTE2: It may be possible to further restrict the permissions in the future to -# be more fine-grained. -# -ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS: false -ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS: false - -# ////////////////////////////////////////////////////////////////////////////////////// -# EMAIL -# - Support for sending emails from fence. Used for user certificates -# and `/google/service_accounts` endpoints -# ////////////////////////////////////////////////////////////////////////////////////// -# Gun Mail Service (for sending emails from fence) -# -# NOTE: Example in comments below -GUN_MAIL: - 'datacommons.io': - smtp_hostname: 'smtp.mailgun.org' - api_key: '' - default_login: 'postmaster@mailgun.example.com' - api_url: 'https://api.mailgun.net/v3/mailgun.example.com' - smtp_password: '' - -# For emails regarding users certificates -EMAIL_SERVER: 'localhost' -SEND_FROM: 'example@gmail.com' -SEND_TO: 'example@gmail.com' - -# ////////////////////////////////////////////////////////////////////////////////////// -# DATA ACCESS: GOOGLE LINKING & SERVICE ACCOUNT REGISTRATION -# - Support `/google/service_accounts` endpoints -# ////////////////////////////////////////////////////////////////////////////////////// -# whether or not to allow access to the /link/google endpoints -ALLOW_GOOGLE_LINKING: true - -# A Google Project with controlled data access will be determined INVALID if -# if it has a parent organization UNLESS that parent organization's ID is in this -# whitelist. -# -# NOTE: Remove the [] and Google Organization IDs if needed. Example in comments below -WHITE_LISTED_GOOGLE_PARENT_ORGS: [] -# - '12345678910' - -# A Google Project with Google Service Accounts determined INVALID will result in the -# the entire project being invalid UNLESS that service accounts's email is in this -# whitelist. -# -# NOTE: Remove the [] and service account emails if needed. Example in comments below -WHITE_LISTED_SERVICE_ACCOUNT_EMAILS: [] -# - 'example@developer.gserviceaccount.com' -# - 'example@test.iam.gserviceaccount.com' - -# when service accounts or google projects are determined invalid, an email is sent -# to the project owners. These settings are for that email -REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION: - enable: false - # this domain MUST exist in GUN_MAIL config - domain: 'example.com' - from: 'do-not-reply@example.com' - subject: 'User service account removal notification' - # the {} gets replaced dynamically in the Python code to be the Project ID - content: > - Service accounts were removed from access control data because some users or - service accounts of GCP Project {} are not authorized to access the data sets - associated to the service accounts, or do not adhere to the security policies. - # this admin email will be included as a recipient to *any* email to anyone about - # service account removal. - # - # WARNING: This is NOT a bcc so the email is visible to the end-user - admin: - - 'admin@example.edu' - -PROBLEM_USER_EMAIL_NOTIFICATION: - # this domain MUST exist in GUN_MAIL config - domain: 'example.com' - from: 'do-not-reply@example.com' - subject: 'Account access error notification' - # the {} gets replaced dynamically in the Python code to be the Project ID - content: > - The Data Commons Framework utilizes dbGaP for data access authorization. - Another member of a Google project you belong to ({}) is attempting to - register a service account to the following additional datasets ({}). - Please contact dbGaP to request access. - # this admin email will be included as a recipient to *any* email to anyone about - # service account removal. - # - # WARNING: This is NOT a bcc so the email is visible to the end-user - admin: - - 'admin@example.edu' - -# Service account email domains that represent a service account that Google owns. -# These are usually created when a sepcific GCP service is enabled. -# This is used for Service Account Validation for Data Access. -GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS: - - 'dataflow-service-producer-prod.iam.gserviceaccount.com' - - 'cloudbuild.gserviceaccount.com' - - 'cloud-ml.google.com.iam.gserviceaccount.com' - - 'container-engine-robot.iam.gserviceaccount.com' - - 'dataflow-service-producer-prod.iam.gserviceaccount.com' - - 'sourcerepo-service-accounts.iam.gserviceaccount.com' - - 'dataproc-accounts.iam.gserviceaccount.com' - - 'gae-api-prod.google.com.iam.gserviceaccount.com' - - 'genomics-api.google.com.iam.gserviceaccount.com' - - 'containerregistry.iam.gserviceaccount.com' - - 'container-analysis.iam.gserviceaccount.com' - - 'cloudservices.gserviceaccount.com' - - 'stackdriver-service.iam.gserviceaccount.com' - - 'appspot.gserviceaccount.com' - - 'partnercontent.gserviceaccount.com' - - 'trifacta-gcloud-prod.iam.gserviceaccount.com' - - 'gcf-admin-robot.iam.gserviceaccount.com' - - 'compute-system.iam.gserviceaccount.com' - - 'gcp-sa-websecurityscanner.iam.gserviceaccount.com' - - 'storage-transfer-service.iam.gserviceaccount.com' - - 'firebase-sa-management.iam.gserviceaccount.com' - - 'firebase-rules.iam.gserviceaccount.com' - - 'gcp-sa-cloudbuild.iam.gserviceaccount.com' - - 'gcp-sa-automl.iam.gserviceaccount.com' - - 'gcp-sa-datalabeling.iam.gserviceaccount.com' - - 'gcp-sa-cloudscheduler.iam.gserviceaccount.com' - -# The types of service accounts that are allowed to be registered at -# /google/service_accounts endpoints -ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: - # compute engine default service account - - 'developer.gserviceaccount.com' - # app engine default service account - - 'appspot.gserviceaccount.com' - # user-managed service account - - 'iam.gserviceaccount.com' - -# Synapse integration and DREAM challenge mapping. Team is from Synapse, and group is -# providing the actual permission in Arborist. User will be added to the group for TTL -# seconds if the team matches. -DREAM_CHALLENGE_TEAM: 'DREAM' -DREAM_CHALLENGE_GROUP: 'DREAM' -SYNAPSE_URI: 'https://repo-prod.prod.sagebase.org/auth/v1' -SYNAPSE_JWKS_URI: -# deprecated, use the discovery_url in the OPENID_CONNECT block for the synapse client -SYNAPSE_DISCOVERY_URL: -SYNAPSE_AUTHZ_TTL: 86400 - -# Role caching for generating presigned urls if max role session increase is true -# then we can increase the amount of time that a session is valid for -MAX_ROLE_SESSION_INCREASE: false -ASSUME_ROLE_CACHE_SECONDS: 1800 - -# Optional user registration feature: Ask users to register (provide firstname/lastname/org/email) on login. -# If user registers, add them to configured Arborist group; idea is that the Arborist group -# will have access to download data. -REGISTER_USERS_ON: false -REGISTERED_USERS_GROUP: '' -# RAS refresh_tokens expire in 15 days -RAS_REFRESH_EXPIRATION: 1296000 -# List of JWT issuers from which Fence will accept GA4GH visas -GA4GH_VISA_ISSUER_ALLOWLIST: - - '{{BASE_URL}}' - - 'https://sts.nih.gov' - - 'https://stsstg.nih.gov' -# Number of projects that can be registered to a Google Service Accont -SERVICE_ACCOUNT_LIMIT: 6 - -# Global sync visas during login -# None(Default): Allow per client i.e. a fence client can pick whether or not to sync their visas during login with parse_visas param in /authorization endpoint -# True: Parse for all clients i.e. a fence client will always sync their visas during login -# False: Parse for no clients i.e. a fence client will not be able to sync visas during login even with parse_visas param -GLOBAL_PARSE_VISAS_ON_LOGIN: -# Settings for usersync with visas -USERSYNC: - sync_from_visas: false - # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed - fallback_to_dbgap_sftp: false - visa_types: - ras: ["https://ras.nih.gov/visas/v1", "https://ras.nih.gov/visas/v1.1"] -RAS_USERINFO_ENDPOINT: '/openid/connect/v1.1/userinfo' \ No newline at end of file diff --git a/helm/fence/fence-google-creds/fence_google_app_creds_secret.json b/helm/fence/fence-google-creds/fence_google_app_creds_secret.json deleted file mode 100644 index e69de29b..00000000 diff --git a/helm/fence/templates/_helpers.tpl b/helm/fence/templates/_helpers.tpl index 315d19b5..1eeaac06 100644 --- a/helm/fence/templates/_helpers.tpl +++ b/helm/fence/templates/_helpers.tpl @@ -69,6 +69,6 @@ Create the name of the service account to use {{- if $localpass }} {{- default (index $localpass.data "postgres-password" | b64dec) }} {{- else }} -{{- default .Values.database.password }} +{{- default .Values.postgres.password }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/fence/templates/db-init.yaml b/helm/fence/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/fence/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/fence/templates/fence-config.yaml b/helm/fence/templates/fence-config.yaml index 355060f5..291a3802 100644 --- a/helm/fence/templates/fence-config.yaml +++ b/helm/fence/templates/fence-config.yaml @@ -4,8 +4,13 @@ metadata: name: fence-config stringData: fence-config.yaml: | + {{- $username := include "gen3.service-postgres" (dict "key" "username" "service" $.Chart.Name "context" $) }} + {{- $password := include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }} + {{- $host := include "gen3.service-postgres" (dict "key" "host" "service" $.Chart.Name "context" $) }} + {{- $port := include "gen3.service-postgres" (dict "key" "port" "service" $.Chart.Name "context" $) }} + {{- $database := include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $) }} BASE_URL: '{{ .Values.FENCE_CONFIG.BASE_URL }}' - DB: 'postgresql://{{ .Values.database.user }}:{{ include "fence.postgres.password" . }}@{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.dbname }}' + DB: 'postgresql://{{ $username }}:{{ $password }}@{{ $host }}:{{ $port }}/{{ $database }}' {{- with .Values.FENCE_CONFIG }} {{- toYaml . | nindent 4 }} {{ end }} diff --git a/helm/fence/templates/fence-creds.yaml b/helm/fence/templates/fence-creds.yaml index ccbffd91..75687b92 100644 --- a/helm/fence/templates/fence-creds.yaml +++ b/helm/fence/templates/fence-creds.yaml @@ -6,10 +6,10 @@ type: Opaque stringData: creds.json: |- { - "db_host": "{{ .Values.database.host }}", - "db_username": "{{ .Values.database.user }}", - "db_password": "{{ .Values.database.password }}", - "db_database": "{{ .Values.database.dbname }}", + "db_host": "{{ include "gen3.service-postgres" (dict "key" "host" "service" $.Chart.Name "context" $) }}", + "db_username": "{{include "gen3.service-postgres" (dict "key" "username" "service" $.Chart.Name "context" $) }}", + "db_password": "{{include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }}", + "db_database": "{{ include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $)}}", "hostname": "{{ .Values.hostname }}", "indexd_password": "", "google_client_secret": "YOUR.GOOGLE.SECRET", diff --git a/helm/fence/templates/fence-deployment.yaml b/helm/fence/templates/fence-deployment.yaml index d5847a00..7df97d02 100644 --- a/helm/fence/templates/fence-deployment.yaml +++ b/helm/fence/templates/fence-deployment.yaml @@ -41,7 +41,7 @@ spec: httpGet: path: /_status port: http - initialDelaySeconds: 30 + initialDelaySeconds: 60 periodSeconds: 60 timeoutSeconds: 30 readinessProbe: @@ -89,7 +89,7 @@ spec: if fence-create migrate --help > /dev/null 2>&1; then if ! grep -E 'ENABLE_DB_MIGRATION"?: *false' /var/www/fence/fence-config.yaml; then echo "Running db migration: fence-create migrate" - cat /var/www/fence/fence-config.yaml | grep DB + cd /fence fence-create migrate else echo "Db migration disabled in fence-config" diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index fc6776af..11a0865f 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -1,6 +1,23 @@ # Default values for fence. # This is a YAML-formatted file. # Declare variables to be passed into your templates. +global: + # Default values are for postgres deployed as a helm chart + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + +db_create: true +postgres: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: fence + username: fence + port: 5432 + # If left empty password will be auto-generated + password: replicaCount: 1 @@ -16,18 +33,7 @@ fullnameOverride: "" hostname: localhost -database: - port: 5432 - host: postgres-postgresql.postgres.svc.cluster.local - - # Credentials used to initialize fence db if it doesn't exist. - master_user: postgres - master_pass: postgres - # Actual fence db creds - user: postgres - password: postgres - dbname: fence serviceAccount: # Specifies whether a service account should be created @@ -304,7 +310,7 @@ privacy_policy: FENCE_CONFIG: APP_NAME: 'Gen3 Data Commons' - BASE_URL: 'http://localhost/user' + BASE_URL: 'https://localhost/user' # A URL-safe base64-encoded 32-byte key for encrypting keys in db diff --git a/helm/gen3-test-data-job/Chart.yaml b/helm/gen3-test-data-job/Chart.yaml index bcf0e126..feeefb98 100644 --- a/helm/gen3-test-data-job/Chart.yaml +++ b/helm/gen3-test-data-job/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: gen3-test-data-job -description: A Helm chart for Kubernetes +description: A Helm chart for generating dummy data in gen3 # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "2022.10" diff --git a/helm/gen3/Chart.lock b/helm/gen3/Chart.lock index abbd081e..597d1273 100644 --- a/helm/gen3/Chart.lock +++ b/helm/gen3/Chart.lock @@ -1,54 +1,57 @@ dependencies: - name: ambassador repository: file://../ambassador - version: 0.1.0 + version: 0.0.1 - name: arborist repository: file://../arborist - version: 0.1.0 + version: 0.0.1 - name: argo-wrapper repository: file://../argo-wrapper - version: 0.1.0 + version: 0.0.1 - name: audit repository: file://../audit - version: 0.1.0 + version: 0.0.1 - name: fence repository: file://../fence - version: 0.1.0 + version: 0.0.1 - name: guppy repository: file://../guppy - version: 0.1.0 + version: 0.0.1 - name: hatchery repository: file://../hatchery - version: 0.1.0 + version: 0.0.1 - name: indexd repository: file://../indexd - version: 0.1.0 + version: 0.0.1 - name: manifestservice repository: file://../manifestservice - version: 0.1.0 + version: 0.0.1 - name: metadata repository: file://../metadata - version: 0.1.0 + version: 0.0.1 - name: peregrine repository: file://../peregrine - version: 0.1.0 + version: 0.0.1 - name: pidgin repository: file://../pidgin - version: 0.1.0 + version: 0.0.1 - name: portal repository: file://../portal - version: 0.1.0 + version: 0.0.1 - name: requestor repository: file://../requestor - version: 0.1.0 + version: 0.0.1 - name: revproxy repository: file://../revproxy - version: 0.1.0 + version: 0.0.1 - name: sheepdog repository: file://../sheepdog - version: 0.1.0 + version: 0.0.1 - name: ssjdispatcher repository: file://../ssjdispatcher - version: 0.1.0 -digest: sha256:7363cb3f2fc0db97142131a851ea7e637772ee83ce1d33164b9886fd4745678a -generated: "2022-09-19T14:49:52.41625-06:00" + version: 0.0.1 +- name: wts + repository: file://../wts + version: 0.0.1 +digest: sha256:94301835cea4dfeb07a462246b7574f9526f0808feb692272f36d0661677cab2 +generated: "2022-10-18T12:52:15.089837-05:00" diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 00ac5541..ad52b512 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -4,74 +4,78 @@ description: An Umbrella Chart to conditionally deploy Gen3 Services #Dependancies dependencies: - - name: ambassador - version: "0.1.0" - repository: "file://../ambassador" - condition: ambassador.enabled - - name: arborist - version: "0.1.0" - repository: "file://../arborist" - condition: arborist.enabled - - name: argo-wrapper - version: "0.1.0" - repository: "file://../argo-wrapper" - condition: argo-wrapper.enabled - - name: audit - version: "0.1.0" - repository: "file://../audit" - condition: audit.enabled - - name: fence - version: "0.1.0" - repository: "file://../fence" - condition: fence.enabled - - name: guppy - version: "0.1.0" - repository: "file://../guppy" - condition: guppy.enabled - - name: hatchery - version: "0.1.0" - repository: "file://../hatchery" - condition: hatchery.enabled - - name: indexd - version: "0.1.0" - repository: "file://../indexd" - condition: indexd.enabled - - name: manifestservice - version: "0.1.0" - repository: "file://../manifestservice" - condition: manifestservice.enabled - - name: metadata - version: "0.1.0" - repository: "file://../metadata" - condition: metadata.enabled - - name: peregrine - version: "0.1.0" - repository: "file://../peregrine" - condition: peregrine.enabled - - name: pidgin - version: "0.1.0" - repository: "file://../pidgin" - condition: pidgin.enabled - - name: portal - version: "0.1.0" - repository: "file://../portal" - condition: portal.enabled - - name: requestor - version: "0.1.0" - repository: "file://../requestor" - condition: requestor.enabled - - name: revproxy - version: "0.1.0" - repository: "file://../revproxy" - condition: revproxy.enabled - - name: sheepdog - version: "0.1.0" - repository: "file://../sheepdog" - condition: sheepdog.enabled - - name: ssjdispatcher - version: "0.1.0" - repository: "file://../ssjdispatcher" - condition: ssjdispatcher.enabled +- name: ambassador + version: "0.0.1" + repository: "file://../ambassador" + condition: ambassador.enabled +- name: arborist + version: "0.0.1" + repository: "file://../arborist" + condition: arborist.enabled +- name: argo-wrapper + version: "0.0.1" + repository: "file://../argo-wrapper" + condition: argo-wrapper.enabled +- name: audit + version: "0.0.1" + repository: "file://../audit" + condition: audit.enabled +- name: fence + version: "0.0.1" + repository: "file://../fence" + condition: fence.enabled +- name: guppy + version: "0.0.1" + repository: "file://../guppy" + condition: guppy.enabled +- name: hatchery + version: "0.0.1" + repository: "file://../hatchery" + condition: hatchery.enabled +- name: indexd + version: "0.0.1" + repository: "file://../indexd" + condition: indexd.enabled +- name: manifestservice + version: "0.0.1" + repository: "file://../manifestservice" + condition: manifestservice.enabled +- name: metadata + version: "0.0.1" + repository: "file://../metadata" + condition: metadata.enabled +- name: peregrine + version: "0.0.1" + repository: "file://../peregrine" + condition: peregrine.enabled +- name: pidgin + version: "0.0.1" + repository: "file://../pidgin" + condition: pidgin.enabled +- name: portal + version: "0.0.1" + repository: "file://../portal" + condition: portal.enabled +- name: requestor + version: "0.0.1" + repository: "file://../requestor" + condition: requestor.enabled +- name: revproxy + version: "0.0.1" + repository: "file://../revproxy" + condition: revproxy.enabled +- name: sheepdog + version: "0.0.1" + repository: "file://../sheepdog" + condition: sheepdog.enabled +- name: ssjdispatcher + version: "0.0.1" + repository: "file://../ssjdispatcher" + condition: ssjdispatcher.enabled +- name: wts + version: "0.0.1" + repository: "file://../wts" + condition: wts.enabled #A chart can be either an 'application' or a 'library' chart. # @@ -86,10 +90,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" \ No newline at end of file +appVersion: "master" \ No newline at end of file diff --git a/helm/gen3/charts/ambassador-0.1.0.tgz b/helm/gen3/charts/ambassador-0.1.0.tgz deleted file mode 100644 index 3ede71a7..00000000 Binary files a/helm/gen3/charts/ambassador-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/arborist-0.1.0.tgz b/helm/gen3/charts/arborist-0.1.0.tgz deleted file mode 100644 index 61105279..00000000 Binary files a/helm/gen3/charts/arborist-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/argo-wrapper-0.1.0.tgz b/helm/gen3/charts/argo-wrapper-0.1.0.tgz deleted file mode 100644 index 734071b6..00000000 Binary files a/helm/gen3/charts/argo-wrapper-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/audit-0.1.0.tgz b/helm/gen3/charts/audit-0.1.0.tgz deleted file mode 100644 index 257fb7cf..00000000 Binary files a/helm/gen3/charts/audit-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/fence-0.1.0.tgz b/helm/gen3/charts/fence-0.1.0.tgz deleted file mode 100644 index f8207594..00000000 Binary files a/helm/gen3/charts/fence-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/guppy-0.1.0.tgz b/helm/gen3/charts/guppy-0.1.0.tgz deleted file mode 100644 index e09f2937..00000000 Binary files a/helm/gen3/charts/guppy-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/hatchery-0.1.0.tgz b/helm/gen3/charts/hatchery-0.1.0.tgz deleted file mode 100644 index 6c629c1f..00000000 Binary files a/helm/gen3/charts/hatchery-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/indexd-0.1.0.tgz b/helm/gen3/charts/indexd-0.1.0.tgz deleted file mode 100644 index d631c9c6..00000000 Binary files a/helm/gen3/charts/indexd-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/manifestservice-0.1.0.tgz b/helm/gen3/charts/manifestservice-0.1.0.tgz deleted file mode 100644 index a40a9b0f..00000000 Binary files a/helm/gen3/charts/manifestservice-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/metadata-0.1.0.tgz b/helm/gen3/charts/metadata-0.1.0.tgz deleted file mode 100644 index b2cece46..00000000 Binary files a/helm/gen3/charts/metadata-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/peregrine-0.1.0.tgz b/helm/gen3/charts/peregrine-0.1.0.tgz deleted file mode 100644 index 4ed8880f..00000000 Binary files a/helm/gen3/charts/peregrine-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/pidgin-0.1.0.tgz b/helm/gen3/charts/pidgin-0.1.0.tgz deleted file mode 100644 index 425ab5b2..00000000 Binary files a/helm/gen3/charts/pidgin-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/portal-0.1.0.tgz b/helm/gen3/charts/portal-0.1.0.tgz deleted file mode 100644 index 10f8d977..00000000 Binary files a/helm/gen3/charts/portal-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/requestor-0.1.0.tgz b/helm/gen3/charts/requestor-0.1.0.tgz deleted file mode 100644 index 6530a74d..00000000 Binary files a/helm/gen3/charts/requestor-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/revproxy-0.1.0.tgz b/helm/gen3/charts/revproxy-0.1.0.tgz deleted file mode 100644 index e0321c33..00000000 Binary files a/helm/gen3/charts/revproxy-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/sheepdog-0.1.0.tgz b/helm/gen3/charts/sheepdog-0.1.0.tgz deleted file mode 100644 index 7425dfe5..00000000 Binary files a/helm/gen3/charts/sheepdog-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/charts/ssjdispatcher-0.1.0.tgz b/helm/gen3/charts/ssjdispatcher-0.1.0.tgz deleted file mode 100644 index a82352a2..00000000 Binary files a/helm/gen3/charts/ssjdispatcher-0.1.0.tgz and /dev/null differ diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index d4e47a40..51c9dc25 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -1,3 +1,11 @@ +global: + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + # Default values for gen3. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -19,7 +27,7 @@ fence: enabled: true guppy: - enabled: true + enabled: false hatchery: enabled: true diff --git a/helm/guppy/Chart.yaml b/helm/guppy/Chart.yaml index 585e5954..6f526c54 100644 --- a/helm/guppy/Chart.yaml +++ b/helm/guppy/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: guppy -description: A Helm chart for Guppy Service +description: A Helm chart for gen3 Guppy Service # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2022.08" +appVersion: "2022.10" diff --git a/helm/hatchery/Chart.yaml b/helm/hatchery/Chart.yaml index 72e6f25b..5d081070 100644 --- a/helm/hatchery/Chart.yaml +++ b/helm/hatchery/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: hatchery -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 Hatchery # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2022.05" +appVersion: "2022.10" diff --git a/helm/indexd/Chart.lock b/helm/indexd/Chart.lock new file mode 100644 index 00000000..9edc4c55 --- /dev/null +++ b/helm/indexd/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:9447ea9a4ddee41221215f9d511d904829f457523bc78ddaa817c161e934f27f +generated: "2022-10-20T21:34:38.85688-05:00" diff --git a/helm/indexd/Chart.yaml b/helm/indexd/Chart.yaml index 576caccc..a80853b8 100644 --- a/helm/indexd/Chart.yaml +++ b/helm/indexd/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: indexd -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 indexd # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,17 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "master" \ No newline at end of file +appVersion: "2022.10" + + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common + condition: db_creation.enabled \ No newline at end of file diff --git a/helm/indexd/templates/_helpers.tpl b/helm/indexd/templates/_helpers.tpl index ccb8ed11..dcb848be 100644 --- a/helm/indexd/templates/_helpers.tpl +++ b/helm/indexd/templates/_helpers.tpl @@ -70,7 +70,7 @@ Create the name of the service account to use {{- if $localpass }} {{- default (index $localpass.data "postgres-password" | b64dec) }} {{- else }} -{{- default .Values.database.password }} +{{- default .Values.postgres.password }} {{- end }} {{- end }} diff --git a/helm/indexd/templates/db-init.yaml b/helm/indexd/templates/db-init.yaml new file mode 100644 index 00000000..7dc1039e --- /dev/null +++ b/helm/indexd/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{ include "common.db-secret" . }} \ No newline at end of file diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml index ecd7ab28..6207db02 100644 --- a/helm/indexd/templates/deployment.yaml +++ b/helm/indexd/templates/deployment.yaml @@ -49,6 +49,9 @@ spec: httpGet: path: /_status port: http + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 30 readinessProbe: httpGet: path: /_status diff --git a/helm/indexd/templates/indexd-secret.yaml b/helm/indexd/templates/indexd-secret.yaml index 18884ebc..b243edbd 100644 --- a/helm/indexd/templates/indexd-secret.yaml +++ b/helm/indexd/templates/indexd-secret.yaml @@ -2,6 +2,9 @@ apiVersion: v1 kind: Secret metadata: name: indexd-secret + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-weight": "-10" type: Opaque data: {{ (.Files.Glob "indexd-secret/*").AsSecrets | indent 2 }} @@ -10,14 +13,17 @@ apiVersion: v1 kind: Secret metadata: name: indexd-creds + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-weight": "-10" type: Opaque stringData: creds.json: |- { - "db_host": {{ .Values.database.host | quote }}, - "db_username": {{ .Values.database.user | quote}}, - "db_password": {{ include "indexd.postgres.password" . | quote }}, - "db_database": {{ .Values.database.dbname | quote }}, + "db_host": "{{ include "gen3.service-postgres" (dict "key" "host" "service" $.Chart.Name "context" $) }}", + "db_username": "{{include "gen3.service-postgres" (dict "key" "username" "service" $.Chart.Name "context" $) }}", + "db_password": "{{include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }}", + "db_database": "{{ include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $)}}", "user_db": { "fence": {{ include "indexd-fence-creds" . | quote }}, "gdcapi": {{ include "indexd-sheepdog-creds" . | quote }}, @@ -30,6 +36,7 @@ kind: Secret metadata: name: indexd-service-creds annotations: + "helm.sh/hook": "pre-install,pre-upgrade" "helm.sh/hook-weight": "-10" type: Opaque stringData: diff --git a/helm/indexd/templates/pre-install.yaml b/helm/indexd/templates/pre-install.yaml index 5e536517..c87f4e9f 100644 --- a/helm/indexd/templates/pre-install.yaml +++ b/helm/indexd/templates/pre-install.yaml @@ -9,6 +9,8 @@ metadata: # creds.json # name: indexd-userdb + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" spec: backoffLimit: 0 template: diff --git a/helm/indexd/values.yaml b/helm/indexd/values.yaml index aa60904c..b50077d2 100644 --- a/helm/indexd/values.yaml +++ b/helm/indexd/values.yaml @@ -1,8 +1,26 @@ - # Default values for indexd. # This is a YAML-formatted file. # Declare variables to be passed into your templates. +global: + # Default values are for postgres deployed as a helm chart + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + +db_create: true +postgres: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: indexd + username: indexd + host: + port: 5432 + # If left empty password will be auto-generated + password: + replicaCount: 1 image: @@ -95,19 +113,6 @@ env: - name: "GEN3_DEBUG" value: "false" -database: - port: 5432 - host: postgres-postgresql.postgres.svc.cluster.local - - # Credentials used to initialize fence db if it doesn't exist. - master_user: postgres - master_pass: postgres - - # Actual fence db creds - user: postgres - password: postgres - dbname: indexd - secrets: userdb: fence: diff --git a/helm/manifestservice/Chart.yaml b/helm/manifestservice/Chart.yaml index 1f440253..7ae2d9cf 100644 --- a/helm/manifestservice/Chart.yaml +++ b/helm/manifestservice/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "2022.10" diff --git a/helm/manifestservice/templates/NOTES.txt b/helm/manifestservice/templates/NOTES.txt index fa4abb0a..70b82c54 100644 --- a/helm/manifestservice/templates/NOTES.txt +++ b/helm/manifestservice/templates/NOTES.txt @@ -1,22 +1 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "manifestservice.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "manifestservice.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "manifestservice.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "manifestservice.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} +{{ .Chart.Name }} has been deployed \ No newline at end of file diff --git a/helm/manifestservice/templates/ingress.yaml b/helm/manifestservice/templates/ingress.yaml deleted file mode 100644 index 68201246..00000000 --- a/helm/manifestservice/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "manifestservice.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "manifestservice.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/manifestservice/values.yaml b/helm/manifestservice/values.yaml index c6122c61..333b82a7 100644 --- a/helm/manifestservice/values.yaml +++ b/helm/manifestservice/values.yaml @@ -18,22 +18,6 @@ serviceAccount: annotations: {} name: "" -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - autoscaling: enabled: false @@ -81,7 +65,6 @@ volumes: terminationGracePeriodSeconds: 50 - env: - name: REQUESTS_CA_BUNDLE value: /etc/ssl/certs/ca-certificates.crt diff --git a/helm/metadata/Chart.lock b/helm/metadata/Chart.lock new file mode 100644 index 00000000..ef02940e --- /dev/null +++ b/helm/metadata/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:9447ea9a4ddee41221215f9d511d904829f457523bc78ddaa817c161e934f27f +generated: "2022-10-20T21:34:40.594362-05:00" diff --git a/helm/metadata/Chart.yaml b/helm/metadata/Chart.yaml index 526a174a..52ddd4a8 100644 --- a/helm/metadata/Chart.yaml +++ b/helm/metadata/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: metadata -description: A Helm chart for Metadata Service +description: A Helm chart for gen3 Metadata Service # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,16 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 2022.05 +appVersion: "2022.10" + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common + condition: db_creation.enabled \ No newline at end of file diff --git a/helm/metadata/templates/_helpers.tpl b/helm/metadata/templates/_helpers.tpl index 817d13b2..8e99ad6d 100644 --- a/helm/metadata/templates/_helpers.tpl +++ b/helm/metadata/templates/_helpers.tpl @@ -71,7 +71,7 @@ Create the name of the service account to use {{- if $localpass }} {{- default (index $localpass.data "postgres-password" | b64dec) }} {{- else }} -{{- default .Values.database.password }} +{{- default .Values.postgres.password }} {{- end }} {{- end }} diff --git a/helm/metadata/templates/db-init.yaml b/helm/metadata/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/metadata/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/metadata/templates/secrets.yaml b/helm/metadata/templates/secrets.yaml index 24cd9a30..fcde0e48 100644 --- a/helm/metadata/templates/secrets.yaml +++ b/helm/metadata/templates/secrets.yaml @@ -7,15 +7,15 @@ stringData: base64Authz.txt: {{ $randomPass | b64enc | quote }} dbcreds.json: | { - "db_host": {{ .Values.database.host | quote }}, - "db_username": {{ .Values.database.user | quote}}, + "db_host": {{ .Values.postgres.host | quote }}, + "db_username": {{ .Values.postgres.user | quote}}, "db_password": {{ include "metadata.postgres.password" . | quote }}, - "db_database": {{ .Values.database.dbname | quote }} + "db_database": {{ .Values.postgres.dbname | quote }} } metadata.env: | DEBUG={{ .Values.debug}} - DB_HOST={{ .Values.database.host }} - DB_USER={{ .Values.database.user }} + DB_HOST={{ .Values.postgres.host }} + DB_USER={{ .Values.postgres.user }} DB_PASSWORD={{ include "metadata.postgres.password" . }} - DB_DATABASE={{ .Values.database.dbname }} + DB_DATABASE={{ .Values.postgres.dbname }} ADMIN_LOGINS={{ $randomPass }} \ No newline at end of file diff --git a/helm/metadata/values.yaml b/helm/metadata/values.yaml index 336cedc7..09169334 100644 --- a/helm/metadata/values.yaml +++ b/helm/metadata/values.yaml @@ -1,7 +1,25 @@ # Default values for metadata. # This is a YAML-formatted file. # Declare variables to be passed into your templates. - +global: + # Default values are for postgres deployed as a helm chart + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + +db_create: true +postgres: + database: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: metadata + username: metadata + host: + port: 5432 + # If left empty password will be auto-generated + password: # Deployment @@ -131,17 +149,3 @@ service: port: 80 targetPort: 80 name: http - -#Configmap -database: - port: 5432 - host: postgres-postgresql.postgres.svc.cluster.local - - # Credentials used to initialize fence db if it doesn't exist. - master_user: postgres - master_pass: postgres - - # Actual fence db creds - user: postgres - password: postgres - dbname: metadata \ No newline at end of file diff --git a/helm/peregrine/Chart.lock b/helm/peregrine/Chart.lock new file mode 100644 index 00000000..ee2e57ed --- /dev/null +++ b/helm/peregrine/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:9447ea9a4ddee41221215f9d511d904829f457523bc78ddaa817c161e934f27f +generated: "2022-10-20T21:12:56.035146-05:00" diff --git a/helm/peregrine/Chart.yaml b/helm/peregrine/Chart.yaml index 7ba5bbb7..6a035831 100644 --- a/helm/peregrine/Chart.yaml +++ b/helm/peregrine/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: peregrine -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 Peregrine service # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,17 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2022.05" +appVersion: "2022.10" + + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common + condition: db_creation.enabled \ No newline at end of file diff --git a/helm/peregrine/peregrine-secret/wsgi.py b/helm/peregrine/peregrine-secret/wsgi.py index aee30ece..0defccd4 100644 --- a/helm/peregrine/peregrine-secret/wsgi.py +++ b/helm/peregrine/peregrine-secret/wsgi.py @@ -5,82 +5,83 @@ from peregrine.api import app, app_init from os import environ -import config_helper +# import config_helper APP_NAME='peregrine' -def load_json(file_name): - return config_helper.load_json(file_name, APP_NAME) +# def load_json(file_name): +# return config_helper.load_json(file_name, APP_NAME) -conf_data = load_json('creds.json') +# conf_data = load_json('creds.json') config = app.config -config["AUTH"] = 'https://auth.service.consul:5000/v3/' -config["AUTH_ADMIN_CREDS"] = None -config["INTERNAL_AUTH"] = None +# config["AUTH"] = 'https://auth.service.consul:5000/v3/' +# config["AUTH_ADMIN_CREDS"] = None +# config["INTERNAL_AUTH"] = None # ARBORIST deprecated, replaced by ARBORIST_URL # ARBORIST_URL is initialized in app_init() directly -config["ARBORIST"] = "http://arborist-service/" +# config["ARBORIST"] = "http://arborist-service/" -# Signpost: deprecated, replaced by index client. -config['SIGNPOST'] = { - 'host': environ.get('SIGNPOST_HOST') or 'http://indexd-service', - 'version': 'v0', - 'auth': ('gdcapi', conf_data.get( 'indexd_password', '{{indexd_password}}')), -} config['INDEX_CLIENT'] = { 'host': environ.get('INDEX_CLIENT_HOST') or 'http://indexd-service', 'version': 'v0', - 'auth': ('gdcapi', conf_data.get( 'indexd_password', '{{indexd_password}}')), + # 'auth': ('gdcapi', environ.get( "PGHOST") ), } -config["FAKE_AUTH"] = False +# config["FAKE_AUTH"] = environ.get( "FAKE_AUTH", False) config["PSQLGRAPH"] = { - 'host': conf_data.get( 'db_host', '{{db_host}}' ), - 'user': conf_data.get( 'db_username', '{{db_username}}' ), - 'password': conf_data.get( 'db_password', '{{db_password}}' ), - 'database': conf_data.get( 'db_database', '{{db_database}}' ), + 'host': environ.get( "PGHOST"), + 'user': environ.get( "PGUSER"), + 'password': environ.get( "PGPASSWORD"), + 'database': environ.get( "PGDB"), } -config['HMAC_ENCRYPTION_KEY'] = conf_data.get( 'hmac_key', '{{hmac_key}}' ) -config['FLASK_SECRET_KEY'] = conf_data.get( 'gdcapi_secret_key', '{{gdcapi_secret_key}}' ) -config['PSQL_USER_DB_CONNECTION'] = 'postgresql://%s:%s@%s:5432/%s' % tuple([ conf_data.get(key, key) for key in ['fence_username', 'fence_password', 'fence_host', 'fence_database']]) +config['HMAC_ENCRYPTION_KEY'] = environ.get( "HMAC_ENCRYPTION_KEY") +config['FLASK_SECRET_KEY'] = environ.get( "FLASK_SECRET_KEY") + +fence_username = environ.get( "FENCE_DB_USER") +fence_password = environ.get( "FENCE_DB_PASS") +fence_host = environ.get( "FENCE_DB_HOST") +fence_database = environ.get( "FENCE_DB_DBNAME") +config['PSQL_USER_DB_CONNECTION'] = 'postgresql://%s:%s@%s:5432/%s' % (fence_username, fence_password, fence_host, fence_database) config['DICTIONARY_URL'] = environ.get('DICTIONARY_URL','https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json') -config['SUBMISSION'] = { - 'bucket': conf_data.get( 'bagit_bucket', '{{bagit_bucket}}' ) -} +# config['SUBMISSION'] = { +# 'bucket': conf_data.get( 'bagit_bucket', '{{bagit_bucket}}' ) +# } -config['STORAGE'] = { - "s3": - { - "access_key": conf_data.get( 's3_access', '{{s3_access}}' ), - 'secret_key': conf_data.get( 's3_secret', '{{s3_secret}}' ) - } -} +# config['STORAGE'] = { +# "s3": +# { +# "access_key": conf_data.get( 's3_access', '{{s3_access}}' ), +# 'secret_key': conf_data.get( 's3_secret', '{{s3_secret}}' ) +# } +# } -config['OIDC_ISSUER'] = 'https://%s/user' % conf_data['hostname'] +hostname = environ.get("CONF_HOSTNAME") +config['OIDC_ISSUER'] = 'https://%s/user' % hostname -config['OAUTH2'] = { - 'client_id': conf_data.get('oauth2_client_id', '{{oauth2_client_id}}'), - 'client_secret': conf_data.get('oauth2_client_secret', '{{oauth2_client_secret}}'), - 'api_base_url': 'https://%s/user/' % conf_data['hostname'], - 'authorize_url': 'https://%s/user/oauth2/authorize' % conf_data['hostname'], - 'access_token_url': 'https://%s/user/oauth2/token' % conf_data['hostname'], - 'refresh_token_url': 'https://%s/user/oauth2/token' % conf_data['hostname'], - 'client_kwargs': { - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % conf_data['hostname'], - 'scope': 'openid data user', - }, - # deprecated key values, should be removed after all commons use new oidc - 'internal_oauth_provider': 'http://fence-service/oauth2/', - 'oauth_provider': 'https://%s/user/oauth2/' % conf_data['hostname'], - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % conf_data['hostname'] -} +# config['OAUTH2'] = { +# 'client_id': conf_data.get('oauth2_client_id', '{{oauth2_client_id}}'), +# 'client_secret': conf_data.get('oauth2_client_secret', '{{oauth2_client_secret}}'), +# 'api_base_url': 'https://%s/user/' % conf_data['hostname'], +# 'authorize_url': 'https://%s/user/oauth2/authorize' % hostname, +# 'access_token_url': 'https://%s/user/oauth2/token' % hostname, +# 'refresh_token_url': 'https://%s/user/oauth2/token' % hostname, +# 'client_kwargs': { +# 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname, +# 'scope': 'openid data user', +# }, +# # deprecated key values, should be removed after all commons use new oidc +# 'internal_oauth_provider': 'http://fence-service/oauth2/', +# 'oauth_provider': 'https://%s/user/oauth2/' % hostname, +# 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname +# } config['USER_API'] = environ.get('FENCE_URL') or 'http://fence-service/' # use the USER_API URL instead of the public issuer URL to accquire JWT keys config['FORCE_ISSUER'] = True +print(config) app_init(app) application = app application.debug = (environ.get('GEN3_DEBUG') == "True") diff --git a/helm/peregrine/templates/_helpers.tpl b/helm/peregrine/templates/_helpers.tpl index 144b0965..1f786d38 100644 --- a/helm/peregrine/templates/_helpers.tpl +++ b/helm/peregrine/templates/_helpers.tpl @@ -66,11 +66,12 @@ Create the name of the service account to use Postgres Password lookup */}} {{- define "peregrine.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} +{{- $masterpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} +# {{- $localpass := (lookup "v1" "Secret" .Release.Namespace "{{ .Chart.Name }}-dbcreds" ) -}} {{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} +{{- default (index $masterpass.data "postgres-password" | b64dec) }} {{- else }} -{{- default .Values.database.password }} +{{- default .Values.postgres.password }} {{- end }} {{- end }} diff --git a/helm/peregrine/templates/db-init.yaml b/helm/peregrine/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/peregrine/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/peregrine/templates/deployment.yaml b/helm/peregrine/templates/deployment.yaml index b1c41b78..a52790da 100644 --- a/helm/peregrine/templates/deployment.yaml +++ b/helm/peregrine/templates/deployment.yaml @@ -38,9 +38,101 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: - {{- toYaml .Values.env | nindent 12 }} + - name: FENCE_DB_USER + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: username + optional: false + - name: FENCE_DB_PASS + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: password + optional: false + - name: FENCE_DB_HOST + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: host + optional: false + - name: FENCE_DB_DBNAME + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: database + optional: false + - name: FLASK_SECRET_KEY + value: "TODO: FIX THIS!!!" + - name: PGHOST + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: host + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: username + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: password + optional: false + - name: PGDB + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: database + optional: false + - name: GEN3_UWSGI_TIMEOUT + value: "600" + - name: DICTIONARY_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: dictionary_url + optional: true + - name: PUBLIC_DATASETS + valueFrom: + configMapKeyRef: + name: manifest-global + key: public_datasets + optional: true + - name: INDEX_CLIENT_HOST + valueFrom: + configMapKeyRef: + name: manifest-global + key: indexd_url + optional: true + - name: GRAPHQL_TIMEOUT + valueFrom: + configMapKeyRef: + name: manifest-peregrine + key: peregrine_timeout + optional: true + - name: FENCE_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: fence_url + optional: true + - name: ARBORIST_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: arborist_url + optional: true + - name: GEN3_SIDECAR + value: "False" volumeMounts: - {{- toYaml .Values.volumeMounts | nindent 12 }} + - name: "config-volume" + readOnly: true + mountPath: "/var/www/peregrine/wsgi.py" + subPath: "wsgi.py" ports: - name: http containerPort: 80 diff --git a/helm/peregrine/templates/peregrine-secret.yaml b/helm/peregrine/templates/peregrine-secret.yaml index 03a2d4e4..fa24590d 100644 --- a/helm/peregrine/templates/peregrine-secret.yaml +++ b/helm/peregrine/templates/peregrine-secret.yaml @@ -4,24 +4,4 @@ metadata: name: peregrine-secret type: Opaque data: -{{ (.Files.Glob "peregrine-secret/*").AsSecrets | indent 2 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: peregrine-creds -type: Opaque -stringData: - creds.json: |- - { - "fence_host": {{ default "fence-postgresql" .Values.secrets.fence.db.host | quote }}, - "fence_username": {{ default "postgres" .Values.secrets.fence.db.username | quote }}, - "fence_password": {{ default .Values.global.postgresql.auth.postgresPassword .Values.secrets.fence.db.password | quote }}, - "fence_database": {{ default .Values.global.postgresql.auth.database .Values.secrets.fence.db.database | quote }}, - "db_host": {{ .Values.database.host | quote }}, - "db_username": {{ .Values.database.user | quote}}, - "db_password": {{ include "peregrine.postgres.password" . | quote }}, - "db_database": {{ .Values.database.dbname | quote }}, - "gdcapi_secret_key": {{ default (randAlphaNum 50) .Values.secrets.gdcapi_secret_key | quote }}, - "hostname": {{ default "ingress.local" .Values.hostname | quote}} - } +{{ (.Files.Glob "peregrine-secret/*").AsSecrets | indent 2 }} \ No newline at end of file diff --git a/helm/peregrine/values.yaml b/helm/peregrine/values.yaml index cfd04c0a..da48976a 100644 --- a/helm/peregrine/values.yaml +++ b/helm/peregrine/values.yaml @@ -1,16 +1,24 @@ -global: - postgresql: - auth: - postgresPassword: "password!123" - database: "peregrine" - -postgres: - enabled: false - # Default values for peregrine. # This is a YAML-formatted file. -# Declare variables to be passed into your templates. - +global: + postgres: + master: + host: postgres-postgresql.postgres.svc.cluster.local + username: postgres + port: 5432 + # If password is left empty the lookup function will look for postgres master password + password: + +db_create: true +postgres: + host: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: peregrine + username: peregrine + port: 5432 + # If left empty password will be auto-generated + password: + replicaCount: 1 image: @@ -49,24 +57,6 @@ service: type: ClusterIP port: 80 -ingress: - enabled: true - className: "" - annotations: { - nginx.ingress.kubernetes.io/rewrite-target: /$2, - kubernetes.io/ingress.class: "nginx" - } - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: ingress.local - paths: - - path: /peregrine(/|$)(.*) - pathType: Prefix - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local resources: {} # We usually recommend not to specify default resources and to leave this as a conscious @@ -95,57 +85,8 @@ affinity: {} env: -- name: GEN3_UWSGI_TIMEOUT - value: "600" -- name: DICTIONARY_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: dictionary_url - optional: true -#needed to be adjusted to use the gen3 umbrella chart or local var ^ -#adding a var in helpers.tpl for later- Elise -- name: PUBLIC_DATASETS - valueFrom: - configMapKeyRef: - name: manifest-global - key: public_datasets - optional: true -# Signpost is deprecated; replace this w INDEX_CLIENT_HOST block -- name: SIGNPOST_HOST - valueFrom: - configMapKeyRef: - name: manifest-global - key: indexd_url - optional: true -- name: INDEX_CLIENT_HOST - valueFrom: - configMapKeyRef: - name: manifest-global - key: indexd_url - optional: true -- name: GRAPHQL_TIMEOUT - valueFrom: - configMapKeyRef: - name: manifest-peregrine - key: peregrine_timeout - optional: true -- name: FENCE_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: fence_url - optional: true -- name: ARBORIST_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: arborist_url - optional: true -- name: GEN3_DEBUG - value: "False" -- name: GEN3_SIDECAR - value: "False" + + volumes: - name: shared-data @@ -153,46 +94,7 @@ volumes: - name: config-volume secret: secretName: "peregrine-secret" -- name: creds-volume - secret: - secretName: "peregrine-creds" -- name: config-helper - configMap: - name: config-helper volumeMounts: -- name: "shared-data" - mountPath: "/var/run/gen3" -- name: "config-volume" - readOnly: true - mountPath: "/var/www/peregrine/wsgi.py" - subPath: "wsgi.py" -- name: "creds-volume" - readOnly: true - mountPath: "/var/www/peregrine/creds.json" - subPath: creds.json -- name: "config-volume" - readOnly: true - mountPath: "/var/www/peregrine/config_helper.py" - subPath: config_helper.py - -secrets: - fence: - db: - host: - user: - pass: - - -database: - port: 5432 - host: postgres-postgresql.postgres.svc.cluster.local - # Credentials used to initialize fence db if it doesn't exist. - master_user: postgres - master_pass: postgres - # Actual fence db creds - user: postgres - password: postgres - dbname: peregrine \ No newline at end of file diff --git a/helm/pidgin/Chart.lock b/helm/pidgin/Chart.lock new file mode 100644 index 00000000..274a5f2b --- /dev/null +++ b/helm/pidgin/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:9447ea9a4ddee41221215f9d511d904829f457523bc78ddaa817c161e934f27f +generated: "2022-10-20T21:34:43.297641-05:00" diff --git a/helm/pidgin/Chart.yaml b/helm/pidgin/Chart.yaml index 27ec77c2..53cfa799 100644 --- a/helm/pidgin/Chart.yaml +++ b/helm/pidgin/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: pidgin -description: A Helm chart for Pidgin Service +description: A Helm chart for gen3 Pidgin Service # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,17 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2022.08" +appVersion: "2022.10" + + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common + condition: db_creation.enabled \ No newline at end of file diff --git a/helm/pidgin/templates/db-init.yaml b/helm/pidgin/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/pidgin/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/pidgin/values.yaml b/helm/pidgin/values.yaml index e83442b8..9144a563 100644 --- a/helm/pidgin/values.yaml +++ b/helm/pidgin/values.yaml @@ -1,7 +1,23 @@ # Default values for pidgin. # This is a YAML-formatted file. # Declare variables to be passed into your templates. - +global: + # Default values are for postgres deployed as a helm chart + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + +db_create: true +postgres: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: pidgin + username: pidgin + port: 5432 + # If left empty password will be auto-generated + password: # Deployment diff --git a/helm/portal/Chart.yaml b/helm/portal/Chart.yaml index 6beaf776..36633f3a 100644 --- a/helm/portal/Chart.yaml +++ b/helm/portal/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: portal -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 data-portal # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "2022.10" diff --git a/helm/portal/templates/configMap.yaml b/helm/portal/templates/configMap.yaml new file mode 100644 index 00000000..ab730274 --- /dev/null +++ b/helm/portal/templates/configMap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: docker-command +data: + dockerStart.sh: + #!/bin/sh + # + # Little startup script fetches data dictionary, and runs the relay + # compiler, then webpack before launching nginx + # + set -eu + + export NODE_ENV=production; + + bash runWebpack.sh + + /usr/sbin/nginx -g 'daemon off;' \ No newline at end of file diff --git a/helm/portal/templates/deployment.yaml b/helm/portal/templates/deployment.yaml index 89e16d57..9331869d 100644 --- a/helm/portal/templates/deployment.yaml +++ b/helm/portal/templates/deployment.yaml @@ -31,19 +31,19 @@ spec: # imagePullPolicy: Never # livenessProbe: # httpGet: - # path: /portal/ + # path: / # port: 80 # initialDelaySeconds: 60 # periodSeconds: 30 # timeoutSeconds: 30 # failureThreshold: 30 - # readinessProbe: - # httpGet: - # path: /portal/ - # port: 80 - # initialDelaySeconds: 30 - # periodSeconds: 60 - # timeoutSeconds: 30 + readinessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 30 # TODO: re-add resources ports: - containerPort: 80 @@ -52,6 +52,8 @@ spec: - /bin/bash - ./dockerStart.sh env: + - name: NODE_ENV + value: dev - name: HOSTNAME value: revproxy-service # disable npm 7's brand new update notifier to prevent Portal from stuck at starting up diff --git a/helm/portal/values.yaml b/helm/portal/values.yaml index 6674734a..26324ef5 100644 --- a/helm/portal/values.yaml +++ b/helm/portal/values.yaml @@ -112,16 +112,6 @@ affinity: automountServiceAccountToken: false -volumes: - - name: config-volume - secret: - secretName: "portal-config" - - name: sponsor-img-volume - secret: - secretName: "portal-sponsor-config" - - name: privacy-policy - configMap: - name: "privacy-policy" resources: @@ -134,9 +124,26 @@ resources: portalApp: "dev" +volumes: + - name: docker-config + configMap: + name: docker-command + - name: config-volume + secret: + secretName: "portal-config" + - name: sponsor-img-volume + secret: + secretName: "portal-sponsor-config" + - name: privacy-policy + configMap: + name: "privacy-policy" + env: volumeMounts: + - name: "docker-config" + mountPath: "/data-portal/dockerStart.sh" + subPath: "dockerStart.sh" - name: "config-volume" mountPath: "/data-portal/data/config/gitops.json" subPath: "gitops.json" diff --git a/helm/requestor/Chart.lock b/helm/requestor/Chart.lock new file mode 100644 index 00000000..2d4691a8 --- /dev/null +++ b/helm/requestor/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:9447ea9a4ddee41221215f9d511d904829f457523bc78ddaa817c161e934f27f +generated: "2022-10-20T21:34:44.799398-05:00" diff --git a/helm/requestor/Chart.yaml b/helm/requestor/Chart.yaml index 46614c48..77d1dee4 100644 --- a/helm/requestor/Chart.yaml +++ b/helm/requestor/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: requestor -description: A Helm chart for Requestor Service +description: A Helm chart for gen3 Requestor Service # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,17 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2022.08" +appVersion: "2022.10" + + +dependencies: +- name: common + version: 0.0.1 + repository: file://../common + condition: db_creation.enabled \ No newline at end of file diff --git a/helm/requestor/templates/db-init.yaml b/helm/requestor/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/requestor/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/requestor/templates/deployment.yaml b/helm/requestor/templates/deployment.yaml index 84de4733..ccad623c 100644 --- a/helm/requestor/templates/deployment.yaml +++ b/helm/requestor/templates/deployment.yaml @@ -70,7 +70,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} initContainers: - - name: {{ .Values.initContainerName }} + - name: requestor-db-migrate image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.volumeMounts }} @@ -81,8 +81,8 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - command: {{ .Values.command }} - {{- with .Values.args }} - args: - {{- toYaml . | nindent 12}} - {{- end }} \ No newline at end of file + command: ["/bin/sh"] + args: + - "-c" + - | + /env/bin/alembic upgrade head \ No newline at end of file diff --git a/helm/requestor/templates/requestor-secret.yaml b/helm/requestor/templates/requestor-secret.yaml index ecedcefe..eec42442 100644 --- a/helm/requestor/templates/requestor-secret.yaml +++ b/helm/requestor/templates/requestor-secret.yaml @@ -13,15 +13,14 @@ stringData: } requestor-config.yaml: |- - { - # Server + # Server - DEBUG: true + DEBUG: true - # Database + # Database - DB_HOST: {{ .Values.secrets.host | quote }}, - DB_USER: {{ .Values.secrets.user | quote }}, - DB_PASSWORD: {{ include "requestor.postgres.password" . | quote }}, - DB_DATABASE: {{ .Values.secrets.database | quote }} - } \ No newline at end of file + DB_HOST: {{ .Values.secrets.host | quote }} + DB_USER: {{ .Values.secrets.user | quote }} + DB_PASSWORD: {{ include "requestor.postgres.password" . | quote }} + DB_DATABASE: {{ .Values.secrets.database | quote }} + \ No newline at end of file diff --git a/helm/requestor/values.yaml b/helm/requestor/values.yaml index 4ff5db46..0c3c4af3 100644 --- a/helm/requestor/values.yaml +++ b/helm/requestor/values.yaml @@ -1,7 +1,23 @@ # Default values for requestor. # This is a YAML-formatted file. # Declare variables to be passed into your templates. - +global: + # Default values are for postgres deployed as a helm chart + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + +db_create: true +postgres: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: requestor + username: requestor + port: 5432 + # If left empty password will be auto-generated + password: # Deployment @@ -108,5 +124,5 @@ service: secrets: host: postgres-postgresql.postgres.svc.cluster.local user: postgres - password: postgres + password: database: requestor \ No newline at end of file diff --git a/helm/revproxy/Chart.yaml b/helm/revproxy/Chart.yaml index 2b4aa4d1..42fc5592 100644 --- a/helm/revproxy/Chart.yaml +++ b/helm/revproxy/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: revproxy -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 revproxy # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "2022.10" diff --git a/helm/revproxy/templates/ingress.yaml b/helm/revproxy/templates/ingress.yaml index 3f06f51d..b7850dba 100644 --- a/helm/revproxy/templates/ingress.yaml +++ b/helm/revproxy/templates/ingress.yaml @@ -4,7 +4,7 @@ metadata: name: revproxy spec: rules: - - host: localhost + - host: {{ default .Values.global.hostname .Values.hostname }} http: paths: - path: / diff --git a/helm/sheepdog/Chart.lock b/helm/sheepdog/Chart.lock new file mode 100644 index 00000000..71060338 --- /dev/null +++ b/helm/sheepdog/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:a25c79b74ec6d89ca5c732e4222f8726ed02aa6a4a21f376afc499e53696c9b5 +generated: "2022-10-20T21:34:46.434949-05:00" diff --git a/helm/sheepdog/Chart.yaml b/helm/sheepdog/Chart.yaml index 300898c4..744f45d3 100644 --- a/helm/sheepdog/Chart.yaml +++ b/helm/sheepdog/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sheepdog -description: A Helm chart for Sheepdog Service +description: A Helm chart for gen3 Sheepdog Service # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,15 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "master" +appVersion: "2022.10" + +dependencies: + - name: common + version: 0.0.1 + repository: file://../common \ No newline at end of file diff --git a/helm/sheepdog/sheepdog-secret/wsgi.py b/helm/sheepdog/sheepdog-secret/wsgi.py index 6d00d0c8..17ce0c00 100644 --- a/helm/sheepdog/sheepdog-secret/wsgi.py +++ b/helm/sheepdog/sheepdog-secret/wsgi.py @@ -5,68 +5,85 @@ from sheepdog.api import app, app_init from os import environ -import config_helper +# import config_helper APP_NAME='sheepdog' -def load_json(file_name): - return config_helper.load_json(file_name, APP_NAME) +# def load_json(file_name): +# return config_helper.load_json(file_name, APP_NAME) -conf_data = load_json('creds.json') +# conf_data = load_json('creds.json') config = app.config -config["AUTH"] = 'https://auth.service.consul:5000/v3/' -config["AUTH_ADMIN_CREDS"] = None -config["INTERNAL_AUTH"] = None +# config["AUTH"] = 'https://auth.service.consul:5000/v3/' +# config["AUTH_ADMIN_CREDS"] = None +# config["INTERNAL_AUTH"] = None # ARBORIST deprecated, replaced by ARBORIST_URL # ARBORIST_URL is initialized in app_init() directly -config["ARBORIST"] = "http://arborist-service/" +# config["ARBORIST"] = "http://arborist-service/" -# Signpost: deprecated, replaced by index client. -config['SIGNPOST'] = { - 'host': environ.get('SIGNPOST_HOST') or 'http://indexd-service', - 'version': 'v0', - 'auth': ('gdcapi', environ.get('INDEXD_PASS')), -} config['INDEX_CLIENT'] = { 'host': environ.get('INDEX_CLIENT_HOST') or 'http://indexd-service', 'version': 'v0', - 'auth': ('gdcapi', environ.get('INDEXD_PASS')), + 'auth': (environ.get( "INDEXD_USER", 'gdcapi'), environ.get( "INDEXD_PASS") ), } -config["FAKE_AUTH"] = False + config["PSQLGRAPH"] = { - 'host': conf_data['db_host'], - 'user': conf_data['db_username'], - 'password': conf_data['db_password'], - 'database': conf_data['db_database'], + 'host': environ.get( "PGHOST"), + 'user': environ.get( "PGUSER"), + 'password': environ.get( "PGPASSWORD"), + 'database': environ.get( "PGDB"), } -config['HMAC_ENCRYPTION_KEY'] = conf_data.get('hmac_key', '{{hmac_key}}') -config['FLASK_SECRET_KEY'] = conf_data.get('gdcapi_secret_key', '{{gdcapi_secret_key}}') -config['PSQL_USER_DB_CONNECTION'] = 'postgresql://%s:%s@%s:5432/%s' % tuple([ conf_data.get(key, key) for key in ['fence_username', 'fence_password', 'fence_host', 'fence_database']]) -config['OIDC_ISSUER'] = 'https://%s/user' % conf_data['hostname'] +config['HMAC_ENCRYPTION_KEY'] = environ.get( "HMAC_ENCRYPTION_KEY") +config['FLASK_SECRET_KEY'] = environ.get( "FLASK_SECRET_KEY") + +fence_username = environ.get( "FENCE_DB_USER") +fence_password = environ.get( "FENCE_DB_PASS") +fence_host = environ.get( "FENCE_DB_HOST") +fence_database = environ.get( "FENCE_DB_DBNAME") +config['PSQL_USER_DB_CONNECTION'] = 'postgresql://%s:%s@%s:5432/%s' % (fence_username, fence_password, fence_host, fence_database) + +config['DICTIONARY_URL'] = environ.get('DICTIONARY_URL','https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json') + +# config['SUBMISSION'] = { +# 'bucket': conf_data.get( 'bagit_bucket', '{{bagit_bucket}}' ) +# } + +# config['STORAGE'] = { +# "s3": +# { +# "access_key": conf_data.get( 's3_access', '{{s3_access}}' ), +# 'secret_key': conf_data.get( 's3_secret', '{{s3_secret}}' ) +# } +# } + +# NOT BEING USED? +hostname = environ.get("CONF_HOSTNAME", "localhost") +config['OIDC_ISSUER'] = 'https://%s/user' % hostname +# TODO: REMOVE THIS??? config['OAUTH2'] = { - 'client_id': conf_data.get('oauth2_client_id', '{{oauth2_client_id}}'), - 'client_secret': conf_data.get('oauth2_client_secret', '{{oauth2_client_secret}}'), - 'api_base_url': 'https://%s/user/' % conf_data['hostname'], - 'authorize_url': 'https://%s/user/oauth2/authorize' % conf_data['hostname'], - 'access_token_url': 'https://%s/user/oauth2/token' % conf_data['hostname'], - 'refresh_token_url': 'https://%s/user/oauth2/token' % conf_data['hostname'], + 'client_id': "conf_data.get('oauth2_client_id', '{{oauth2_client_id}}')", + 'client_secret': "conf_data.get('oauth2_client_secret', '{{oauth2_client_secret}}')", + 'api_base_url': 'https://%s/user/' % hostname, + 'authorize_url': 'https://%s/user/oauth2/authorize' % hostname, + 'access_token_url': 'https://%s/user/oauth2/token' % hostname, + 'refresh_token_url': 'https://%s/user/oauth2/token' % hostname, 'client_kwargs': { - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % conf_data['hostname'], + 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname, 'scope': 'openid data user', }, # deprecated key values, should be removed after all commons use new oidc 'internal_oauth_provider': 'http://fence-service/oauth2/', - 'oauth_provider': 'https://%s/user/oauth2/' % conf_data['hostname'], - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % conf_data['hostname'] + 'oauth_provider': 'https://%s/user/oauth2/' % hostname, + 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname } + config['USER_API'] = environ.get('FENCE_URL') or 'http://fence-service/' # use the USER_API URL instead of the public issuer URL to accquire JWT keys -config['FORCE_ISSUER'] = True -config['DICTIONARY_URL'] = environ.get('DICTIONARY_URL','https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json') - +# config['FORCE_ISSUER'] = True +print(config) app_init(app) application = app -application.debug = (environ.get('GEN3_DEBUG') == "True") \ No newline at end of file +application.debug = (environ.get('GEN3_DEBUG') == "True") diff --git a/helm/sheepdog/sheepdog-secret/wsgi_copy.py b/helm/sheepdog/sheepdog-secret/wsgi_copy.py new file mode 100644 index 00000000..6d00d0c8 --- /dev/null +++ b/helm/sheepdog/sheepdog-secret/wsgi_copy.py @@ -0,0 +1,72 @@ +##################################################### +# DO NOT CHANGE THIS FILE # +# config updates should be done in the service code # +##################################################### + +from sheepdog.api import app, app_init +from os import environ +import config_helper + +APP_NAME='sheepdog' +def load_json(file_name): + return config_helper.load_json(file_name, APP_NAME) + +conf_data = load_json('creds.json') +config = app.config + +config["AUTH"] = 'https://auth.service.consul:5000/v3/' +config["AUTH_ADMIN_CREDS"] = None +config["INTERNAL_AUTH"] = None + +# ARBORIST deprecated, replaced by ARBORIST_URL +# ARBORIST_URL is initialized in app_init() directly +config["ARBORIST"] = "http://arborist-service/" + +# Signpost: deprecated, replaced by index client. +config['SIGNPOST'] = { + 'host': environ.get('SIGNPOST_HOST') or 'http://indexd-service', + 'version': 'v0', + 'auth': ('gdcapi', environ.get('INDEXD_PASS')), +} +config['INDEX_CLIENT'] = { + 'host': environ.get('INDEX_CLIENT_HOST') or 'http://indexd-service', + 'version': 'v0', + 'auth': ('gdcapi', environ.get('INDEXD_PASS')), +} +config["FAKE_AUTH"] = False +config["PSQLGRAPH"] = { + 'host': conf_data['db_host'], + 'user': conf_data['db_username'], + 'password': conf_data['db_password'], + 'database': conf_data['db_database'], +} + +config['HMAC_ENCRYPTION_KEY'] = conf_data.get('hmac_key', '{{hmac_key}}') +config['FLASK_SECRET_KEY'] = conf_data.get('gdcapi_secret_key', '{{gdcapi_secret_key}}') +config['PSQL_USER_DB_CONNECTION'] = 'postgresql://%s:%s@%s:5432/%s' % tuple([ conf_data.get(key, key) for key in ['fence_username', 'fence_password', 'fence_host', 'fence_database']]) +config['OIDC_ISSUER'] = 'https://%s/user' % conf_data['hostname'] + +config['OAUTH2'] = { + 'client_id': conf_data.get('oauth2_client_id', '{{oauth2_client_id}}'), + 'client_secret': conf_data.get('oauth2_client_secret', '{{oauth2_client_secret}}'), + 'api_base_url': 'https://%s/user/' % conf_data['hostname'], + 'authorize_url': 'https://%s/user/oauth2/authorize' % conf_data['hostname'], + 'access_token_url': 'https://%s/user/oauth2/token' % conf_data['hostname'], + 'refresh_token_url': 'https://%s/user/oauth2/token' % conf_data['hostname'], + 'client_kwargs': { + 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % conf_data['hostname'], + 'scope': 'openid data user', + }, + # deprecated key values, should be removed after all commons use new oidc + 'internal_oauth_provider': 'http://fence-service/oauth2/', + 'oauth_provider': 'https://%s/user/oauth2/' % conf_data['hostname'], + 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % conf_data['hostname'] +} +config['USER_API'] = environ.get('FENCE_URL') or 'http://fence-service/' +# use the USER_API URL instead of the public issuer URL to accquire JWT keys +config['FORCE_ISSUER'] = True +config['DICTIONARY_URL'] = environ.get('DICTIONARY_URL','https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json') + +app_init(app) +application = app +application.debug = (environ.get('GEN3_DEBUG') == "True") \ No newline at end of file diff --git a/helm/sheepdog/templates/db-init.yaml b/helm/sheepdog/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/sheepdog/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/sheepdog/templates/deployment.yaml b/helm/sheepdog/templates/deployment.yaml index 6cdd6030..095fb200 100644 --- a/helm/sheepdog/templates/deployment.yaml +++ b/helm/sheepdog/templates/deployment.yaml @@ -72,6 +72,74 @@ spec: # - "-c" # - "sleep infinity" env: + - name: FENCE_DB_USER + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: username + optional: false + - name: FENCE_DB_PASS + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: password + optional: false + - name: FENCE_DB_HOST + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: host + optional: false + - name: FENCE_DB_DBNAME + valueFrom: + secretKeyRef: + name: fence-dbcreds + key: database + optional: false + - name: FLASK_SECRET_KEY + value: "TODO: FIX THIS!!!" + - name: PGHOST + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: host + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: username + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: password + optional: false + - name: PGDB + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: database + optional: false + - name: PUBLIC_DATASETS + valueFrom: + configMapKeyRef: + name: manifest-global + key: public_datasets + optional: true + # - name: INDEX_CLIENT_HOST + # valueFrom: + # configMapKeyRef: + # name: manifest-global + # key: indexd_url + # optional: true + - name: FENCE_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: fence_url + optional: true - name: INDEXD_PASS valueFrom: secretKeyRef: @@ -98,13 +166,13 @@ spec: - name: AUTH_NAMESPACE value: {{ . }} {{- end }} - - name: REQUESTS_CA_BUNDLE - # - # override python 'requests' SSL certificate bundle - # to use system trusted certs - # which includes our private certificate authority - # - value: /etc/ssl/certs/ca-certificates.crt + # - name: REQUESTS_CA_BUNDLE + # # + # # override python 'requests' SSL certificate bundle + # # to use system trusted certs + # # which includes our private certificate authority + # # + # value: /etc/ssl/certs/ca-certificates.crt - name: GEN3_DEBUG value: "True" {{- with .Values.ddTraceEnabled }} diff --git a/helm/sheepdog/templates/pre-install.yaml b/helm/sheepdog/templates/pre-install.yaml index a3a688ef..5ec2f0c5 100644 --- a/helm/sheepdog/templates/pre-install.yaml +++ b/helm/sheepdog/templates/pre-install.yaml @@ -1,59 +1,70 @@ apiVersion: batch/v1 kind: Job metadata: - name: gdcdb-create + name: sheepdog-dbinit + annotations: + "helm.sh/hook": "pre-install" #,pre-upgrade" spec: backoffLimit: 0 template: metadata: - annotations: - # This is what defines this resource as a hook. Without this line, the - # job is considered part of the release. - "helm.sh/hook": pre-install - "helm.sh/hook-weight": "-5" - "helm.sh/hook-delete-policy": hook-succeeded labels: app: gen3job spec: automountServiceAccountToken: false - volumes: - - name: creds-volume - secret: - secretName: "sheepdog-creds" + # volumes: + # - name: creds-volume + # secret: + # secretName: "sheepdog-creds" containers: - name: sheepdog image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" env: - name: DICTIONARY_URL value: {{ .Values.dictionaryUrl }} - volumeMounts: - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/sheepdog/creds.json" - subPath: creds.json - imagePullPolicy: Always + - name: PGHOST + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: host + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: username + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: password + optional: false + - name: PGDB + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: database + optional: false + # volumeMounts: + # - name: "creds-volume" + # readOnly: true + # mountPath: "/var/www/sheepdog/creds.json" + # subPath: creds.json + # imagePullPolicy: Always command: ["/bin/bash" ] args: - "-c" # Script always succeeds if it runs (echo exits with 0) - | - eval $(python 2> /dev/null <=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "ssjdispatcher.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/terraform-runner-job/Chart.yaml b/helm/terraform-runner-job/Chart.yaml index 60d84467..a8b74404 100644 --- a/helm/terraform-runner-job/Chart.yaml +++ b/helm/terraform-runner-job/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: terraform-runner-job -description: A Helm chart for Kubernetes +description: A Helm chart for provisioning prequisites cloud resources for gen3 # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "feat_tf1X_jq" diff --git a/helm/terraform-runner-job/templates/jobs.yaml b/helm/terraform-runner-job/templates/jobs.yaml index fd0dd774..9238bd05 100644 --- a/helm/terraform-runner-job/templates/jobs.yaml +++ b/helm/terraform-runner-job/templates/jobs.yaml @@ -24,11 +24,8 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} env: - - name: gen3Env - valueFrom: - configMapKeyRef: - name: global - key: environment + - name: USE_TF_1 + value: "True" - name: JENKINS_HOME value: "devterm" - name: GEN3_HOME @@ -53,7 +50,7 @@ spec: export LC_ALL=C.UTF-8 export LANG=C.UTF-8 source "${GEN3_HOME}/gen3/gen3setup.sh" - gen3 workon default "${resouce_name}_${workspace_name}" + gen3 workon default "helm" gen3 cd cp /config.tfvars ./config.tfvars {{ if .Values.terraform.destroy }} diff --git a/helm/wts/Chart.lock b/helm/wts/Chart.lock index b995cbfd..d251d368 100644 --- a/helm/wts/Chart.lock +++ b/helm/wts/Chart.lock @@ -1,6 +1,6 @@ dependencies: -- name: db-setup - repository: file://../db-setup - version: 0.1.0 -digest: sha256:ebe24f94c9bc192b19cebb4b52d53702323feddb93c21facccc99147b748ff10 -generated: "2022-09-16T12:58:43.964836-04:00" +- name: common + repository: file://../common + version: 0.0.1 +digest: sha256:a25c79b74ec6d89ca5c732e4222f8726ed02aa6a4a21f376afc499e53696c9b5 +generated: "2022-10-20T21:34:47.899465-05:00" diff --git a/helm/wts/Chart.yaml b/helm/wts/Chart.yaml index 632378f6..1b71326d 100644 --- a/helm/wts/Chart.yaml +++ b/helm/wts/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: wts -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 workspace token service # A chart can be either an 'application' or a 'library' chart. # @@ -15,15 +15,15 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "2022.10" dependencies: - - name: db-setup - version: 0.1.0 - repository: file://../db-setup \ No newline at end of file + - name: common + version: 0.0.1 + repository: file://../common \ No newline at end of file diff --git a/helm/wts/charts/db-setup-0.1.0.tgz b/helm/wts/charts/db-setup-0.1.0.tgz deleted file mode 100644 index 80036fa4..00000000 Binary files a/helm/wts/charts/db-setup-0.1.0.tgz and /dev/null differ diff --git a/helm/wts/templates/_helpers.tpl b/helm/wts/templates/_helpers.tpl index 7dd6486d..d13a9fca 100644 --- a/helm/wts/templates/_helpers.tpl +++ b/helm/wts/templates/_helpers.tpl @@ -69,6 +69,6 @@ Create the name of the service account to use {{- if $localpass }} {{- default (index $localpass.data "postgres-password" | b64dec) }} {{- else }} -{{- default .Values.database.password }} +{{- default .Values.postgres.password }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/wts/templates/appInit.yaml b/helm/wts/templates/appInit.yaml deleted file mode 100644 index 40edce2e..00000000 --- a/helm/wts/templates/appInit.yaml +++ /dev/null @@ -1 +0,0 @@ -{{- include "db-setup.-setup-job" . -}} \ No newline at end of file diff --git a/helm/wts/templates/db-init.yaml b/helm/wts/templates/db-init.yaml new file mode 100644 index 00000000..e53cb144 --- /dev/null +++ b/helm/wts/templates/db-init.yaml @@ -0,0 +1,3 @@ +{{- include "common.db-setup-job" . }} +--- +{{- include "common.db-secret" . }} diff --git a/helm/wts/templates/deployment.yaml b/helm/wts/templates/deployment.yaml index 0df03cb7..72c1c6be 100644 --- a/helm/wts/templates/deployment.yaml +++ b/helm/wts/templates/deployment.yaml @@ -60,10 +60,6 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - - name: "wts-secret" - readOnly: true - mountPath: "/var/www/wts/dbcreds.json" - subPath: dbcreds.json - name: "wts-secret" readOnly: true mountPath: "/var/www/wts/appcreds.json" @@ -83,12 +79,36 @@ spec: path: /_status port: 80 env: - - name: POSTGRES_CREDS_FILE - value: "/var/www/wts/dbcreds.json" - name: SECRET_CONFIG value: "/var/www/wts/appcreds.json" - name: AUTH_PLUGINS value: k8s + - name: PGHOST + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: host + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: username + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: password + optional: false + - name: PGDB + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: database + optional: false + - name: SQLALCHEMY_DATABASE_URI + value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB) resources: {{- toYaml .Values.resources | nindent 12 }} initContainers: @@ -96,17 +116,37 @@ spec: imagePullPolicy: Always image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" volumeMounts: - - name: "wts-secret" - readOnly: true - mountPath: "/var/www/wts/dbcreds.json" - subPath: dbcreds.json - name: "wts-secret" readOnly: true mountPath: "/var/www/wts/appcreds.json" subPath: appcreds.json env: - - name: POSTGRES_CREDS_FILE - value: "/var/www/wts/dbcreds.json" + - name: PGHOST + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: host + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: username + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: password + optional: false + - name: PGDB + valueFrom: + secretKeyRef: + name: {{ .Chart.Name }}-dbcreds + key: database + optional: false + - name: SQLALCHEMY_DATABASE_URI + value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB) - name: SECRET_CONFIG value: "/var/www/wts/appcreds.json" resources: diff --git a/helm/wts/templates/secret.yaml b/helm/wts/templates/secret.yaml index 738d1b05..76901964 100644 --- a/helm/wts/templates/secret.yaml +++ b/helm/wts/templates/secret.yaml @@ -24,15 +24,8 @@ stringData: "wts_base_url": "https://{{ .Values.hostname }}/wts/", "encryption_key": "{{ .Values.encpryption_key | default (randAlphaNum 32 | b64enc) }}", "secret_key": "{{ .Values.secret_key | default (randAlphaNum 32 | b64enc) }}", - "fence_base_url": "http://fence-service/", + "fence_base_url": "https://localhost/user", "oidc_client_id": "{{ .Values.oidc_client_id | default (randAlphaNum 32) }}", "oidc_client_secret": "{{ .Values.oidc_client_secret | default (randAlphaNum 32) }}", "external_oidc": [] - } - dbcreds.json: | - { - "db_host": {{ .Values.database.host | quote }}, - "db_username": {{ .Values.database.user | quote}}, - "db_password": {{ include "wts.postgres.password" . | quote }}, - "db_database": {{ .Values.database.dbname | quote }} } \ No newline at end of file diff --git a/helm/wts/values.yaml b/helm/wts/values.yaml index d9586cfb..382f9980 100644 --- a/helm/wts/values.yaml +++ b/helm/wts/values.yaml @@ -1,3 +1,24 @@ +global: + # Default values are for postgres deployed as a helm chart + postgres: + host: postgres-postgresql.postgres.svc.cluster.local + master: + username: postgres + password: + port: 5432 + + +# Whether or not to run database creation job +# The job is idempotant +db_creation: true +postgres: + # If db does not exist in postgres cluster and db_creation is set ot true then these databases will be created for you + database: wts + username: wts + # If left empty it will be auto-generated + password: + + # Default values for wts. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -5,18 +26,20 @@ replicaCount: 1 image: - repository: quay.io/cdis/workspace-token-service - pullPolicy: Always + repository: quay.io/cdis/wts + pullPolicy: Never # Overrides the image tag whose default is the chart appVersion. - tag: "2022.09" + tag: "jqtest" imagePullSecrets: [] nameOverride: "" fullnameOverride: "" + + hostname: localhost oidc_client_id: -oidc_client_secret: +oidc_client_secret: serviceAccount: # Specifies whether a service account should be created @@ -50,13 +73,6 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 -database: - port: 5432 - host: postgres-postgresql.postgres.svc.cluster.local - - user: postgres - password: - dbname: wts service: type: ClusterIP @@ -108,15 +124,3 @@ secrets: # "db_database": "wts_default" # } -postgres: - host: postgres-postgresql.postgres.svc.cluster.local - - master: - username: postgres - password: TEnTIpzG2I - port: 5432 - - databases: - - databaseName: wts - username: wts - password: wts_password \ No newline at end of file diff --git a/wip/acronymbot/Chart.yaml b/wip/acronymbot/Chart.yaml index ba43e43c..162210aa 100644 --- a/wip/acronymbot/Chart.yaml +++ b/wip/acronymbot/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: acronymbot -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 acronymbot # A chart can be either an 'application' or a 'library' chart. # diff --git a/wip/cogwheel/Chart.yaml b/wip/cogwheel/Chart.yaml index b492e5eb..87fb8367 100644 --- a/wip/cogwheel/Chart.yaml +++ b/wip/cogwheel/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: cogwheel -description: A Helm chart for Kubernetes +description: A Helm chart for gen3 cogwheel # A chart can be either an 'application' or a 'library' chart. # @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2022.07" +appVersion: "2022.10"