Need to mount the configuration files in "requestor" and "metadata" d…

…irectories to accommodate the new Dockerfile for these services.

We are also using Poetry to manage virtual environments instead of python, so the command to run alembic needs to be modified.

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-09-09T20:23:45Z",
+  "generated_at": "2024-11-05T18:59:51Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -476,7 +476,7 @@
         "hashed_secret": "d84ce25b0f9bc2cc263006ae39453efb22cc2900",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 71,
+        "line_number": 70,
         "type": "Secret Keyword"
       },
       {

helm/gen3/Chart.yaml

@@ -56,7 +56,7 @@ dependencies:
   repository: "file://../manifestservice"
   condition: manifestservice.enabled
 - name: metadata
-  version: 0.1.13
+  version: 0.1.14
   repository: "file://../metadata"
   condition: metadata.enabled
 - name: peregrine
@@ -72,7 +72,7 @@ dependencies:
   repository: "file://../portal"
   condition: portal.enabled
 - name: requestor
-  version: 0.1.12
+  version: 0.1.13
   repository: "file://../requestor"
   condition: requestor.enabled
 - name: revproxy
@@ -128,7 +128,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.44
+version: 0.1.45
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/gen3/README.md

@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.43](https://img.shields.io/badge/Version-0.1.43-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.45](https://img.shields.io/badge/Version-0.1.45-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -31,12 +31,12 @@ Helm chart to deploy Gen3 Data Commons
 | file://../hatchery | hatchery | 0.1.10 |
 | file://../indexd | indexd | 0.1.15 |
 | file://../manifestservice | manifestservice | 0.1.15 |
-| file://../metadata | metadata | 0.1.13 |
+| file://../metadata | metadata | 0.1.14 |
 | file://../neuvector | neuvector | 0.1.0 |
 | file://../peregrine | peregrine | 0.1.14 |
 | file://../pidgin | pidgin | 0.1.11 |
-| file://../portal | portal | 0.1.17 |
-| file://../requestor | requestor | 0.1.12 |
+| file://../portal | portal | 0.1.18 |
+| file://../requestor | requestor | 0.1.13 |
 | file://../revproxy | revproxy | 0.1.17 |
 | file://../sheepdog | sheepdog | 0.1.15 |
 | file://../sower | sower | 0.1.13 |

helm/metadata/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.13
+version: 0.1.14
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/metadata/README.md

@@ -1,6 +1,6 @@
 # metadata
 
-![Version: 0.1.13](https://img.shields.io/badge/Version-0.1.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Metadata Service
 
@@ -25,7 +25,7 @@ A Helm chart for gen3 Metadata Service
 | affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. |
 | aggMdsConfig | string | `"{\n  \"configuration\": {\n    \"schema\": {\n      \"_subjects_count\": {\n        \"type\": \"integer\"\n      },\n      \"__manifest\": {\n        \"description\": \"an array of filename (usually DRS ids and its size\",\n        \"type\": \"array\",\n        \"properties\": {\n          \"file_name\": {\n            \"type\": \"string\"\n          },\n          \"file_size\": {\n            \"type\": \"integer\"\n          }\n        }\n      },\n      \"tags\": {\n        \"type\": \"array\"\n      },\n      \"_unique_id\": {},\n      \"study_description\": {},\n      \"study_id\": {},\n      \"study_url\": {},\n      \"project_id\": {},\n      \"short_name\": {\n        \"default\": \"not_set\"\n      },\n      \"year\": {\n        \"default\": \"not_set\"\n      },\n      \"full_name\": {},\n      \"commons_url\": {},\n      \"commons\": {}\n    },\n    \"settings\": {\n      \"cache_drs\": true\n    }\n  },\n  \"adapter_commons\": {\n    \"Gen3\": {\n      \"mds_url\": \"https://gen3.datacommons.io/\",\n      \"commons_url\": \"gen3.datacommons.io/\",\n      \"adapter\": \"gen3\",\n      \"config\": {\n        \"guid_type\": \"discovery_metadata\",\n        \"study_field\": \"gen3_discovery\"\n      },\n      \"keep_original_fields\": false,\n      \"field_mappings\": {\n        \"tags\": \"path:tags\",\n        \"_unique_id\": \"path:_unique_id\",\n        \"study_description\": \"path:summary\",\n        \"full_name\": \"path:study_title\",\n        \"short_name\": \"path:short_name\",\n        \"year\": \"path:year\",\n        \"accession_number\": \"path:accession_number\",\n        \"commons\": \"Gen3 Data Commons\",\n        \"study_url\": {\n          \"path\": \"link\",\n          \"default\": \"unknown\"\n        }\n      }\n    }\n  }\n}\n"` |  |
 | aggMdsNamespace | string | `"default"` | Namespae to use if AggMds is enabled. |
-| args | list | `["-c","/env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
+| args | list | `["-c","poetry run alembic upgrade head || /env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
 | automountServiceAccountToken | bool | `false` | Automount the default service account token |
 | autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas |
 | autoscaling.enabled | bool | `false` | Whether autoscaling is enabled |
@@ -84,7 +84,7 @@ A Helm chart for gen3 Metadata Service
 | initResources.limits | map | `{"cpu":0.8,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use |
 | initResources.limits.cpu | string | `0.8` | The maximum amount of CPU the container can use |
 | initResources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use |
-| initVolumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"}]` | Volumes to mount to the init container. |
+| initVolumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/mds/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"}]` | Volumes to mount to the init container. |
 | metricsEnabled | bool | `false` | Whether Metrics are enabled. |
 | partOf | string | `"Discovery-Tab"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. |
 | postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you |
@@ -119,5 +119,5 @@ A Helm chart for gen3 Metadata Service
 | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. |
 | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. |
 | useAggMds | bool | `"True"` | Set to true to aggregate metadata from multiple other Metadata Service instances. |
-| volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. |
+| volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/mds/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. |
 

helm/metadata/values.yaml

@@ -253,6 +253,10 @@ volumeMounts:
     readOnly: true
     mountPath: /src/.env
     subPath: metadata.env
+  - name: config-volume-g3auto
+    readOnly: true
+    mountPath: /mds/.env
+    subPath: metadata.env
   - name: config-volume
     readOnly: true
     mountPath: /aggregate_config.json
@@ -286,6 +290,10 @@ initVolumeMounts:
     readOnly: true
     mountPath: /src/.env
     subPath: metadata.env
+  - name: config-volume-g3auto
+    readOnly: true
+    mountPath: /mds/.env
+    subPath: metadata.env
 # -- (map) Resource limits for the init container.
 initResources:
   # -- (map) The maximum amount of resources that the container is allowed to use
@@ -300,7 +308,7 @@ command: ["/bin/sh"]
 args:
   - "-c"
   - |
-    /env/bin/alembic upgrade head
+    poetry run alembic upgrade head || /env/bin/alembic upgrade head
 
 # Service and Pod
 serviceAnnotations:

helm/portal/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.18
+version: 0.1.19
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/requestor/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.12
+version: 0.1.13
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/requestor/README.md

@@ -1,6 +1,6 @@
 # requestor
 
-![Version: 0.1.12](https://img.shields.io/badge/Version-0.1.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.13](https://img.shields.io/badge/Version-0.1.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Requestor Service
 
@@ -23,7 +23,7 @@ A Helm chart for gen3 Requestor Service
 | affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["requestor"]` | Value for the match expression key. |
 | affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. |
 | arboristUrl | string | `"http://arborist-service"` | Arborist service URL. |
-| args | list | `["-c","/env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
+| args | list | `["-c","poetry run alembic upgrade head || /env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
 | automountServiceAccountToken | bool | `false` | Automount the default service account token |
 | autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas |
 | autoscaling.enabled | bool | `false` | Whether autoscaling is enabled |

helm/requestor/values.yaml

@@ -190,6 +190,10 @@ volumeMounts:
   #   readOnly: true
   #   mountPath: "/src/requestor-config.yaml"
   #   subPath: "requestor-config.yaml"
+  # - name: "config-volume"
+  #   readOnly: true
+  #   mountPath: "/requestor/requestor-config.yaml"
+  #   subPath: "requestor-config.yaml"
 
 # -- (map) Resource requests and limits for the containers in the pod
 resources:
@@ -223,7 +227,7 @@ command: ["/bin/sh"]
 args:
   - "-c"
   - |
-    /env/bin/alembic upgrade head
+    poetry run alembic upgrade head || /env/bin/alembic upgrade head
 
 # Service and Pod
 # -- (map) Kubernetes service information.