feat(non-aws-s3-buckets): Added support for non AWS s3 buckets for usersync

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2023-11-20T21:39:41Z",
+  "generated_at": "2024-02-09T13:25:05Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -301,7 +301,7 @@
         "hashed_secret": "9d8fada0e01336e865c461bb3549084d206fe6da",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 181,
+        "line_number": 182,
         "type": "Secret Keyword"
       }
     ],
@@ -365,7 +365,7 @@
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 1916,
+        "line_number": 1918,
         "type": "Secret Keyword"
       }
     ],
@@ -374,7 +374,7 @@
         "hashed_secret": "1740c48fa3141d4851b14f97e3bc0f46f7670672",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 127,
+        "line_number": 128,
         "type": "Secret Keyword"
       }
     ],
@@ -383,7 +383,7 @@
         "hashed_secret": "9b5925ea817163740dfb287a9894e8ab3aba2c18",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 216,
+        "line_number": 218,
         "type": "Secret Keyword"
       }
     ],

helm/fence/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.14
+version: 0.1.15
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/fence/README.md

@@ -1,6 +1,6 @@
 # fence
 
-![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.15](https://img.shields.io/badge/Version-0.1.15-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Fence
 
@@ -164,10 +164,11 @@ A Helm chart for gen3 Fence
 | serviceAccount.create | bool | `true` | Specifies whether a service account should be created. |
 | serviceAccount.name | string | `"fence-sa"` | The name of the service account |
 | tolerations | list | `[]` | Tolerations for the pods |
-| usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. |
+| usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"s3Endpoint":"https://s3.amazonaws.com","schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. |
 | usersync.addDbgap | bool | `false` | Force attempting a dbgap sync if "true", falls back on user.yaml |
 | usersync.custom_image | string | `nil` | To set a custom image for pulling the user.yaml file from S3. Default is the Gen3 Awshelper image. |
 | usersync.onlyDbgap | bool | `false` | Forces ONLY a dbgap sync if "true", IGNORING user.yaml |
+| usersync.s3Endpoint | string | `"https://s3.amazonaws.com"` | The endpoint url of your s3 service |
 | usersync.schedule | string | `"*/30 * * * *"` | The cron schedule expression to use in the usersync cronjob. Runs every 30 minutes by default. |
 | usersync.secrets | map | `{"awsAccessKeyId":"","awsSecretAccessKey":""}` | Secret information |
 | usersync.secrets.awsAccessKeyId | str | `""` | AWS access key ID for usersync S3 bucket |

helm/fence/templates/usersync-cron.yaml

@@ -158,6 +158,8 @@ spec:
                     key: hostname
             - name: userYamlS3Path
               value: {{ .Values.usersync.userYamlS3Path | quote }}
+            - name: s3Endpoint
+              value: {{ .Values.usersync.s3Endpoint | quote }}
             - name: slackWebHook
               value: {{ .Values.usersync.slack_webhook | quote }}
             volumeMounts:
@@ -185,7 +187,7 @@ spec:
                   n=0
                   until [ $n -ge 5 ]; do
                     echo "Download attempt $n"
-                    aws s3 cp "${userYamlS3Path}" /mnt/shared/user.yaml && break
+                    aws s3 cp "${userYamlS3Path}" /mnt/shared/user.yaml --endpoint-url $s3Endpoint && break
                     n=$[$n+1]
                     sleep 2
                     echo "test 1"
@@ -208,4 +210,4 @@ spec:
                 fi
                 echo "Helper exit ok"
           restartPolicy: "Never"
-{{- end }}
+{{- end }}

helm/fence/values.yaml

@@ -102,6 +102,8 @@ usersync:
   onlyDbgap: false
   # -- (string) Path to the user.yaml file in S3.
   userYamlS3Path: s3://cdis-gen3-users/helm-test/user.yaml
+  # -- (string) The endpoint url of your s3 service
+  s3Endpoint: https://s3.amazonaws.com
   # -- (string) Slack webhook endpoint used with certain jobs.
   slack_webhook: None
   # -- (bool) Will echo what files we are seeing on dbgap ftp to Slack.

helm/gen3/Chart.yaml

@@ -115,7 +115,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.21
+version: 0.1.22
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/gen3/README.md

@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.21](https://img.shields.io/badge/Version-0.1.21-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.22](https://img.shields.io/badge/Version-0.1.22-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -80,10 +80,11 @@ Helm chart to deploy Gen3 Data Commons
 | fence.image | map | `{"repository":null,"tag":null}` | Docker image information. |
 | fence.image.repository | string | `nil` | The Docker image repository for the fence service. |
 | fence.image.tag | string | `nil` | Overrides the image tag whose default is the chart appVersion. |
-| fence.usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. |
+| fence.usersync | map | `{"addDbgap":false,"custom_image":null,"onlyDbgap":false,"s3Endpoint":"https://s3.amazonaws.com","schedule":"*/30 * * * *","secrets":{"awsAccessKeyId":"","awsSecretAccessKey":""},"slack_send_dbgap":false,"slack_webhook":"None","syncFromDbgap":false,"userYamlS3Path":"s3://cdis-gen3-users/helm-test/user.yaml","usersync":false}` | Configuration options for usersync cronjob. |
 | fence.usersync.addDbgap | bool | `false` | Force attempting a dbgap sync if "true", falls back on user.yaml |
 | fence.usersync.custom_image | string | `nil` | To set a custom image for pulling the user.yaml file from S3. Default is the Gen3 Awshelper image. |
 | fence.usersync.onlyDbgap | bool | `false` | Forces ONLY a dbgap sync if "true", IGNORING user.yaml |
+| fence.usersync.s3Endpoint | string | `"https://s3.amazonaws.com"` | The endpoint url of your s3 service |
 | fence.usersync.schedule | string | `"*/30 * * * *"` | The cron schedule expression to use in the usersync cronjob. Runs every 30 minutes by default. |
 | fence.usersync.secrets | map | `{"awsAccessKeyId":"","awsSecretAccessKey":""}` | Secret information |
 | fence.usersync.secrets.awsAccessKeyId | str | `""` | AWS access key ID for usersync S3 bucket |

helm/gen3/values.yaml

@@ -145,6 +145,8 @@ fence:
     onlyDbgap: false
     # -- (string) Path to the user.yaml file in S3.
     userYamlS3Path: s3://cdis-gen3-users/helm-test/user.yaml
+    # -- (string) The endpoint url of your s3 service
+    s3Endpoint: https://s3.amazonaws.com
     # -- (string) Slack webhook endpoint used with certain jobs.
     slack_webhook: None
     # -- (bool) Will echo what files we are seeing on dbgap ftp to Slack.