From 0886a904cd1b5b791129bd3e67a0818a1f955ec0 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Fri, 9 Feb 2024 07:25:14 -0600 Subject: [PATCH] feat(non-aws-s3-buckets): Added support for non AWS s3 buckets for usersync --- .secrets.baseline | 10 +++++----- helm/fence/Chart.yaml | 2 +- helm/fence/README.md | 5 +++-- helm/fence/templates/usersync-cron.yaml | 6 ++++-- helm/fence/values.yaml | 2 ++ helm/gen3/Chart.yaml | 2 +- helm/gen3/README.md | 5 +++-- helm/gen3/values.yaml | 2 ++ 8 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index fc20cd69..8c785020 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2023-11-20T21:39:41Z", + "generated_at": "2024-02-09T13:25:05Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -301,7 +301,7 @@ "hashed_secret": "9d8fada0e01336e865c461bb3549084d206fe6da", "is_secret": false, "is_verified": false, - "line_number": 181, + "line_number": 182, "type": "Secret Keyword" } ], @@ -365,7 +365,7 @@ "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db", "is_secret": false, "is_verified": false, - "line_number": 1916, + "line_number": 1918, "type": "Secret Keyword" } ], @@ -374,7 +374,7 @@ "hashed_secret": "1740c48fa3141d4851b14f97e3bc0f46f7670672", "is_secret": false, "is_verified": false, - "line_number": 127, + "line_number": 128, "type": "Secret Keyword" } ], @@ -383,7 +383,7 @@ "hashed_secret": "9b5925ea817163740dfb287a9894e8ab3aba2c18", "is_secret": false, "is_verified": false, - "line_number": 216, + "line_number": 218, "type": "Secret Keyword" } ], diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index d3203de8..a0d42d8a 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.14 +version: 0.1.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/fence/README.md b/helm/fence/README.md index f80d4215..8c9d5438 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -1,6 +1,6 @@ # fence -![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.15](https://img.shields.io/badge/Version-0.1.15-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Fence @@ -164,10 +164,11 @@ A Helm chart for gen3 Fence | serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | | serviceAccount.name | string | `"fence-sa"` | The name of the service account | | tolerations | list | `[]` | Tolerations for the pods | -| usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. | +| usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"s3Endpoint":"https://s3.amazonaws.com","schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. | | usersync.addDbgap | bool | `false` | Force attempting a dbgap sync if "true", falls back on user.yaml | | usersync.custom_image | string | `nil` | To set a custom image for pulling the user.yaml file from S3. Default is the Gen3 Awshelper image. | | usersync.onlyDbgap | bool | `false` | Forces ONLY a dbgap sync if "true", IGNORING user.yaml | +| usersync.s3Endpoint | string | `"https://s3.amazonaws.com"` | The endpoint url of your s3 service | | usersync.schedule | string | `"*/30 * * * *"` | The cron schedule expression to use in the usersync cronjob. Runs every 30 minutes by default. | | usersync.secrets | map | `{"awsAccessKeyId":"","awsSecretAccessKey":""}` | Secret information | | usersync.secrets.awsAccessKeyId | str | `""` | AWS access key ID for usersync S3 bucket | diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml index ef683944..020cd509 100644 --- a/helm/fence/templates/usersync-cron.yaml +++ b/helm/fence/templates/usersync-cron.yaml @@ -158,6 +158,8 @@ spec: key: hostname - name: userYamlS3Path value: {{ .Values.usersync.userYamlS3Path | quote }} + - name: s3Endpoint + value: {{ .Values.usersync.s3Endpoint | quote }} - name: slackWebHook value: {{ .Values.usersync.slack_webhook | quote }} volumeMounts: @@ -185,7 +187,7 @@ spec: n=0 until [ $n -ge 5 ]; do echo "Download attempt $n" - aws s3 cp "${userYamlS3Path}" /mnt/shared/user.yaml && break + aws s3 cp "${userYamlS3Path}" /mnt/shared/user.yaml --endpoint-url $s3Endpoint && break n=$[$n+1] sleep 2 echo "test 1" @@ -208,4 +210,4 @@ spec: fi echo "Helper exit ok" restartPolicy: "Never" -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index f0352346..f05f35ef 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -102,6 +102,8 @@ usersync: onlyDbgap: false # -- (string) Path to the user.yaml file in S3. userYamlS3Path: s3://cdis-gen3-users/helm-test/user.yaml + # -- (string) The endpoint url of your s3 service + s3Endpoint: https://s3.amazonaws.com # -- (string) Slack webhook endpoint used with certain jobs. slack_webhook: None # -- (bool) Will echo what files we are seeing on dbgap ftp to Slack. diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 259fb435..9cb89637 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -115,7 +115,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.21 +version: 0.1.22 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 220a9cc4..c353682f 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,6 +1,6 @@ # gen3 -![Version: 0.1.21](https://img.shields.io/badge/Version-0.1.21-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.22](https://img.shields.io/badge/Version-0.1.22-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons @@ -80,10 +80,11 @@ Helm chart to deploy Gen3 Data Commons | fence.image | map | `{"repository":null,"tag":null}` | Docker image information. | | fence.image.repository | string | `nil` | The Docker image repository for the fence service. | | fence.image.tag | string | `nil` | Overrides the image tag whose default is the chart appVersion. | -| fence.usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. | +| fence.usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"s3Endpoint":"https://s3.amazonaws.com","schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. | | fence.usersync.addDbgap | bool | `false` | Force attempting a dbgap sync if "true", falls back on user.yaml | | fence.usersync.custom_image | string | `nil` | To set a custom image for pulling the user.yaml file from S3. Default is the Gen3 Awshelper image. | | fence.usersync.onlyDbgap | bool | `false` | Forces ONLY a dbgap sync if "true", IGNORING user.yaml | +| fence.usersync.s3Endpoint | string | `"https://s3.amazonaws.com"` | The endpoint url of your s3 service | | fence.usersync.schedule | string | `"*/30 * * * *"` | The cron schedule expression to use in the usersync cronjob. Runs every 30 minutes by default. | | fence.usersync.secrets | map | `{"awsAccessKeyId":"","awsSecretAccessKey":""}` | Secret information | | fence.usersync.secrets.awsAccessKeyId | str | `""` | AWS access key ID for usersync S3 bucket | diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 0bb20303..44c66b02 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -145,6 +145,8 @@ fence: onlyDbgap: false # -- (string) Path to the user.yaml file in S3. userYamlS3Path: s3://cdis-gen3-users/helm-test/user.yaml + # -- (string) The endpoint url of your s3 service + s3Endpoint: https://s3.amazonaws.com # -- (string) Slack webhook endpoint used with certain jobs. slack_webhook: None # -- (bool) Will echo what files we are seeing on dbgap ftp to Slack.