Skip to content

Commit

Permalink
Orchestration cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
jawadqur committed Oct 18, 2022
1 parent 88ddcfc commit 3842940
Show file tree
Hide file tree
Showing 74 changed files with 414 additions and 302 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
postgres.txt
**/charts/
values.yaml
49 changes: 26 additions & 23 deletions .secrets.baseline
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"files": "^.secrets.baseline$",
"lines": null
},
"generated_at": "2022-09-19T14:03:53Z",
"generated_at": "2022-10-18T23:08:35Z",
"plugins_used": [
{
"name": "AWSKeyDetector"
Expand Down Expand Up @@ -63,14 +63,14 @@
"hashed_secret": "04024ac03c114027f1116abedeb4bb78b01a31db",
"is_secret": false,
"is_verified": false,
"line_number": 84,
"line_number": 85,
"type": "Secret Keyword"
},
{
"hashed_secret": "0c9967f3918994e95ab61396a76a7d10f783c8f7",
"is_secret": false,
"is_verified": false,
"line_number": 108,
"line_number": 109,
"type": "Secret Keyword"
}
],
Expand All @@ -95,28 +95,34 @@
"helm/db-setup/templates/_db_setup.tpl": [
{
"hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5",
"is_secret": false,
"is_verified": false,
"line_number": 24,
"line_number": 70,
"type": "Secret Keyword"
}
],
"helm/db-setup/values.yaml": [
{
"hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
"hashed_secret": "a70646783e43f444ba3430a4110bb7bdd65bdb3a",
"is_secret": false,
"is_verified": false,
"line_number": 6,
"line_number": 16,
"type": "Secret Keyword"
},
{
"hashed_secret": "a70646783e43f444ba3430a4110bb7bdd65bdb3a",
"hashed_secret": "874947acc1ffd819b836f6e049b2f1ab8303cb6c",
"is_secret": false,
"is_verified": false,
"line_number": 12,
"line_number": 20,
"type": "Secret Keyword"
},
}
],
"helm/dicom-server/values.yaml": [
{
"hashed_secret": "874947acc1ffd819b836f6e049b2f1ab8303cb6c",
"hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
"is_secret": false,
"is_verified": false,
"line_number": 15,
"line_number": 69,
"type": "Secret Keyword"
}
],
Expand Down Expand Up @@ -274,17 +280,19 @@
],
"helm/peregrine/values.yaml": [
{
"hashed_secret": "c286f6974f94aab4cfaf2ef49ee0465a8495f563",
"hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
"is_secret": false,
"is_verified": false,
"line_number": 4,
"line_number": 170,
"type": "Secret Keyword"
},
}
],
"helm/requestor/values.yaml": [
{
"hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
"is_secret": false,
"is_verified": false,
"line_number": 195,
"line_number": 111,
"type": "Secret Keyword"
}
],
Expand Down Expand Up @@ -344,15 +352,10 @@
],
"helm/wts/values.yaml": [
{
"hashed_secret": "7f8d4ff76d11ef5e9aa633658ed42694b91d3da9",
"is_verified": false,
"line_number": 116,
"type": "Secret Keyword"
},
{
"hashed_secret": "a70646783e43f444ba3430a4110bb7bdd65bdb3a",
"hashed_secret": "206c80413b9a96c1312cc346b7d2517b84463edd",
"is_secret": false,
"is_verified": false,
"line_number": 122,
"line_number": 134,
"type": "Secret Keyword"
}
]
Expand Down
19 changes: 10 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,14 @@ kubectl run postgres-postgresql-client --rm --tty -i --restart='Never' --namespa

Once you get a PSQL shell into postgres create databses by running the following
```
CREATE DATBASE arborist;
CREATE DATBASE audit;
CREATE DATBASE fence;
CREATE DATBASE indexd;
CREATE DATBASE metadata;
CREATE DATBASE peregrine;
CREATE DATBASE sheepdog;
CREATE DATABASE arborist;
CREATE DATABASE audit;
CREATE DATABASE fence;
CREATE DATABASE indexd;
CREATE DATABASE metadata;
CREATE DATABASE peregrine;
CREATE DATABASE sheepdog;
CREATE DATABASE requestor;
```


Expand Down Expand Up @@ -72,10 +73,10 @@ After configuration is complete, take note of the client ID that was created. Yo

## Helm chart deployment

### Install all charts
### Install Gen3
```
cd ./helm
for i in $(ls); do helm upgrade --install $i ./$i; done
helm upgrade --install gen3 ./gen3 -f values.yaml
```

### Install fence with google login secrets
Expand Down
6 changes: 3 additions & 3 deletions helm/ambassador/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: ambassador
description: A Helm chart for Kubernetes
description: A Helm chart for deploying ambassador for gen3

# A chart can be either an 'application' or a 'library' chart.
#
Expand All @@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
version: 0.0.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
appVersion: "1.4.2"
6 changes: 3 additions & 3 deletions helm/arborist/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: arborist
description: A Helm chart for Kubernetes
description: A Helm chart for gen3 arborist

# A chart can be either an 'application' or a 'library' chart.
#
Expand All @@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
version: 0.0.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "2022.05"
appVersion: "2022.10"
6 changes: 3 additions & 3 deletions helm/argo-wrapper/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: argo-wrapper
description: A Helm chart for Argo Wrapper Service
description: A Helm chart for gen3 Argo Wrapper Service

# A chart can be either an 'application' or a 'library' chart.
#
Expand All @@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
version: 0.0.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "master"
appVersion: "2022.10"
4 changes: 2 additions & 2 deletions helm/audit/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
version: 0.0.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
appVersion: 2022.05
appVersion: "2022.10"


4 changes: 2 additions & 2 deletions helm/aws-es-proxy/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: aws-es-proxy
description: A Helm chart for AWS ES Proxy Service
description: A Helm chart for AWS ES Proxy Service for gen3

# A chart can be either an 'application' or a 'library' chart.
#
Expand All @@ -15,7 +15,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
version: 0.0.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
Expand Down
6 changes: 3 additions & 3 deletions helm/db-setup/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: db-setup
description: A Helm chart for Kubernetes
description: A Helm chart for provisioning databases in gen3

# A chart can be either an 'application' or a 'library' chart.
#
Expand All @@ -15,10 +15,10 @@ type: library
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
version: 0.0.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
appVersion: "master"
61 changes: 51 additions & 10 deletions helm/db-setup/templates/_db_setup.tpl
Original file line number Diff line number Diff line change
@@ -1,31 +1,72 @@
{{/*
Postgres Password lookup
*/}}
{{- define "postgres.master.password" -}}
{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}}
{{- if $localpass }}
{{- default (index $localpass.data "postgres-password" | b64dec) }}
{{- else }}
{{- default $.Values.global.postgres.master.password }}
{{- end }}
{{- end }}



{{- define "db-setup.setup-job" -}}
apiVersion: batch/v1
kind: Job
metadata:
name: db-setup
name: {{ .Chart.Name }}-dbcreate
annotations:
"helm.sh/hook": "pre-install,pre-upgrade"
"helm.sh/hook-delete-policy": hook-succeeded
spec:
template:
metadata:
labels:
app: gen3job
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": hook-succeeded
spec:
restartPolicy: OnFailure
containers:
- name: db-setup
image: quay.io/cdis/awshelper:master
imagePullPolicy: Always
command: ["/bin/bash", "-c"]
env:
- name: PGPASSWORD
value: "{{ include "postgres.master.password" . }}"
- name: PGUSER
value: "{{ $.Values.global.postgres.master.username }}"
- name: PGPORT
value: "{{ $.Values.global.postgres.master.port }}"
- name: PGHOST
value: "{{ $.Values.global.postgres.host }}"
args:
- |
export PGPASSWORD="{{ $.Values.postgres.master.password }}"
{{- range .Values.postgres.databases }}
psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -tc "SELECT 1 FROM pg_database WHERE datname = '{{ .databaseName }}'" | grep -q 1 || psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -c "CREATE DATABASE {{ .databaseName }};"
psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -tc "SELECT 1 FROM pg_user WHERE usename = '{{ .username }}'" | grep -q 1 || psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -c "CREATE USER {{ .username }} WITH PASSWORD '{{ .password }}';"
psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -c "GRANT ALL ON DATABASE {{ .databaseName }} TO {{ .username }} WITH GRANT OPTION;"
psql -h {{ $.Values.postgres.host }} -U {{ $.Values.postgres.master.username }} -p {{ $.Values.postgres.master.port }} -d {{ .databaseName }} -c "CREATE EXTENSION ltree; ALTER ROLE {{ .username }} WITH LOGIN"
if psql -lqt | cut -d \| -f 1 | grep -qw {{ .databaseName }}; then
echo "Database named {{ .databaseName }} already exists."
else
psql -tc "SELECT 1 FROM pg_database WHERE datname = '{{ .databaseName }}'" | grep -q 1 || psql -c "CREATE DATABASE {{ .databaseName }};"
psql -tc "SELECT 1 FROM pg_user WHERE usename = '{{ .username }}'" | grep -q 1 || psql -c "CREATE USER {{ .username }} WITH PASSWORD '{{ .password }}';"
psql -c "GRANT ALL ON DATABASE {{ .databaseName }} TO {{ .username }} WITH GRANT OPTION;"
psql -d {{ .databaseName }} -c "CREATE EXTENSION ltree; ALTER ROLE {{ .username }} WITH LOGIN"
fi
{{- end }}
{{- end }}

{{ define "db-setup.secret" }}
{{- range .Values.postgres.databases }}
apiVersion: v1
kind: Secret
metadata:
name: {{ .service }}-dbcreds
annotations:
"helm.sh/hook": "pre-install,pre-upgrade"
"helm.sh/resource-policy": "keep"
stringData:
database: "{{ .databaseName }}"
username: "{{ .username }}"
password: "{{ .password }}"
{{- end -}}
{{- end -}}
23 changes: 14 additions & 9 deletions helm/db-setup/values.yaml
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
global:
postgres:
host: postgres-postgresql.postgres.svc.cluster.local
master:
username: postgres
port: 5432
# If password is left empty the lookup function will look for postgres master password
password:

postgres:
host: postgres-postgresql.postgres.svc.cluster.local

master:
username: postgres
password: postgres
port: 5432

# An array of databases to create.
databases:
- databaseName: wts
- service: wts
databaseName: wts
username: wts
password: wts_password
- databaseName: indexd
- service: indexd
databaseName: indexd
username: indexd
password: indexd_password
2 changes: 1 addition & 1 deletion helm/dicom-server/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: dicom-server
description: A Helm chart for AWS Dicom Server
description: A Helm chart for gen3 Dicom Server

# A chart can be either an 'application' or a 'library' chart.
#
Expand Down
2 changes: 1 addition & 1 deletion helm/dicom-viewer/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: dicom-viewer
description: A Helm chart for AWS Dicom Viewer
description: A Helm chart for gen3 Dicom Viewer

# A chart can be either an 'application' or a 'library' chart.
#
Expand Down
Loading

0 comments on commit 3842940

Please sign in to comment.