Merge pull request #207 from uc-cdis/feat/al-updates

Need to mount the configuration files in "requestor" and "metadata" d…

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-09-09T20:23:45Z",
+  "generated_at": "2024-11-05T18:59:51Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -476,7 +476,7 @@
         "hashed_secret": "d84ce25b0f9bc2cc263006ae39453efb22cc2900",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 71,
+        "line_number": 70,
         "type": "Secret Keyword"
       },
       {

helm/audit/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.13
+version: 0.1.14
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/audit/README.md

@@ -1,6 +1,6 @@
 # audit
 
-![Version: 0.1.13](https://img.shields.io/badge/Version-0.1.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for Kubernetes
 

helm/audit/templates/deployment.yaml

@@ -90,6 +90,11 @@ spec:
                 readOnly: true
                 mountPath: "/src/audit-service-config.yaml"
                 subPath: "audit-service-config.yaml"
+              # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+              - name: "config-volume"
+                readOnly: true
+                mountPath: "/audit/audit-service-config.yaml"
+                subPath: "audit-service-config.yaml"
             {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -134,12 +139,18 @@ spec:
           args:
             - "-c"
             - |
-              /env/bin/alembic upgrade head
+              # Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility 
+              poetry run alembic upgrade head || /env/bin/alembic upgrade head
           volumeMounts:
             - name: "config-volume"
               readOnly: true
               mountPath: "/src/audit-service-config.yaml"
               subPath: "audit-service-config.yaml"
+            # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+            - name: "config-volume"
+              readOnly: true
+              mountPath: "/audit/audit-service-config.yaml"
+              subPath: "audit-service-config.yaml"
             {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

helm/gen3/Chart.yaml

@@ -17,7 +17,7 @@ dependencies:
   repository: "file://../argo-wrapper"
   condition: argo-wrapper.enabled
 - name: audit
-  version: 0.1.13
+  version: 0.1.14
   repository: "file://../audit"
   condition: audit.enabled
 - name: aws-es-proxy
@@ -56,7 +56,7 @@ dependencies:
   repository: "file://../manifestservice"
   condition: manifestservice.enabled
 - name: metadata
-  version: 0.1.13
+  version: 0.1.14
   repository: "file://../metadata"
   condition: metadata.enabled
 - name: peregrine
@@ -68,11 +68,11 @@ dependencies:
   repository: "file://../pidgin"
   condition: pidgin.enabled
 - name: portal
-  version: 0.1.18
+  version: 0.1.19
   repository: "file://../portal"
   condition: portal.enabled
 - name: requestor
-  version: 0.1.12
+  version: 0.1.13
   repository: "file://../requestor"
   condition: requestor.enabled
 - name: revproxy
@@ -92,7 +92,7 @@ dependencies:
   condition: sower.enabled
   repository: "file://../sower"
 - name: wts
-  version: 0.1.14
+  version: 0.1.15
   repository: "file://../wts"
   condition: wts.enabled
 
@@ -128,7 +128,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.44
+version: 0.1.45
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/gen3/README.md

@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.43](https://img.shields.io/badge/Version-0.1.43-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.45](https://img.shields.io/badge/Version-0.1.45-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -21,7 +21,7 @@ Helm chart to deploy Gen3 Data Commons
 | file://../ambassador | ambassador | 0.1.12 |
 | file://../arborist | arborist | 0.1.12 |
 | file://../argo-wrapper | argo-wrapper | 0.1.8 |
-| file://../audit | audit | 0.1.13 |
+| file://../audit | audit | 0.1.14 |
 | file://../aws-es-proxy | aws-es-proxy | 0.1.10 |
 | file://../common | common | 0.1.14 |
 | file://../etl | etl | 0.1.3 |
@@ -31,17 +31,17 @@ Helm chart to deploy Gen3 Data Commons
 | file://../hatchery | hatchery | 0.1.10 |
 | file://../indexd | indexd | 0.1.15 |
 | file://../manifestservice | manifestservice | 0.1.15 |
-| file://../metadata | metadata | 0.1.13 |
+| file://../metadata | metadata | 0.1.14 |
 | file://../neuvector | neuvector | 0.1.0 |
 | file://../peregrine | peregrine | 0.1.14 |
 | file://../pidgin | pidgin | 0.1.11 |
-| file://../portal | portal | 0.1.17 |
-| file://../requestor | requestor | 0.1.12 |
+| file://../portal | portal | 0.1.19 |
+| file://../requestor | requestor | 0.1.13 |
 | file://../revproxy | revproxy | 0.1.17 |
 | file://../sheepdog | sheepdog | 0.1.15 |
 | file://../sower | sower | 0.1.13 |
 | file://../ssjdispatcher | ssjdispatcher | 0.1.11 |
-| file://../wts | wts | 0.1.14 |
+| file://../wts | wts | 0.1.15 |
 | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 |
 | https://helm.elastic.co | elasticsearch | 7.10.2 |
 

helm/metadata/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.13
+version: 0.1.14
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/metadata/README.md

@@ -1,6 +1,6 @@
 # metadata
 
-![Version: 0.1.13](https://img.shields.io/badge/Version-0.1.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Metadata Service
 
@@ -25,7 +25,7 @@ A Helm chart for gen3 Metadata Service
 | affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. |
 | aggMdsConfig | string | `"{\n  \"configuration\": {\n    \"schema\": {\n      \"_subjects_count\": {\n        \"type\": \"integer\"\n      },\n      \"__manifest\": {\n        \"description\": \"an array of filename (usually DRS ids and its size\",\n        \"type\": \"array\",\n        \"properties\": {\n          \"file_name\": {\n            \"type\": \"string\"\n          },\n          \"file_size\": {\n            \"type\": \"integer\"\n          }\n        }\n      },\n      \"tags\": {\n        \"type\": \"array\"\n      },\n      \"_unique_id\": {},\n      \"study_description\": {},\n      \"study_id\": {},\n      \"study_url\": {},\n      \"project_id\": {},\n      \"short_name\": {\n        \"default\": \"not_set\"\n      },\n      \"year\": {\n        \"default\": \"not_set\"\n      },\n      \"full_name\": {},\n      \"commons_url\": {},\n      \"commons\": {}\n    },\n    \"settings\": {\n      \"cache_drs\": true\n    }\n  },\n  \"adapter_commons\": {\n    \"Gen3\": {\n      \"mds_url\": \"https://gen3.datacommons.io/\",\n      \"commons_url\": \"gen3.datacommons.io/\",\n      \"adapter\": \"gen3\",\n      \"config\": {\n        \"guid_type\": \"discovery_metadata\",\n        \"study_field\": \"gen3_discovery\"\n      },\n      \"keep_original_fields\": false,\n      \"field_mappings\": {\n        \"tags\": \"path:tags\",\n        \"_unique_id\": \"path:_unique_id\",\n        \"study_description\": \"path:summary\",\n        \"full_name\": \"path:study_title\",\n        \"short_name\": \"path:short_name\",\n        \"year\": \"path:year\",\n        \"accession_number\": \"path:accession_number\",\n        \"commons\": \"Gen3 Data Commons\",\n        \"study_url\": {\n          \"path\": \"link\",\n          \"default\": \"unknown\"\n        }\n      }\n    }\n  }\n}\n"` |  |
 | aggMdsNamespace | string | `"default"` | Namespae to use if AggMds is enabled. |
-| args | list | `["-c","/env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
+| args | list | `["-c","# Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility\npoetry run alembic upgrade head || /env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
 | automountServiceAccountToken | bool | `false` | Automount the default service account token |
 | autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas |
 | autoscaling.enabled | bool | `false` | Whether autoscaling is enabled |
@@ -84,7 +84,7 @@ A Helm chart for gen3 Metadata Service
 | initResources.limits | map | `{"cpu":0.8,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use |
 | initResources.limits.cpu | string | `0.8` | The maximum amount of CPU the container can use |
 | initResources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use |
-| initVolumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"}]` | Volumes to mount to the init container. |
+| initVolumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/mds/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"}]` | Volumes to mount to the init container. |
 | metricsEnabled | bool | `false` | Whether Metrics are enabled. |
 | partOf | string | `"Discovery-Tab"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. |
 | postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you |
@@ -119,5 +119,5 @@ A Helm chart for gen3 Metadata Service
 | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. |
 | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. |
 | useAggMds | bool | `"True"` | Set to true to aggregate metadata from multiple other Metadata Service instances. |
-| volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. |
+| volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/mds/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. |
 

helm/metadata/values.yaml

@@ -253,6 +253,11 @@ volumeMounts:
     readOnly: true
     mountPath: /src/.env
     subPath: metadata.env
+  # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+  - name: config-volume-g3auto
+    readOnly: true
+    mountPath: /mds/.env
+    subPath: metadata.env
   - name: config-volume
     readOnly: true
     mountPath: /aggregate_config.json
@@ -286,6 +291,11 @@ initVolumeMounts:
     readOnly: true
     mountPath: /src/.env
     subPath: metadata.env
+  # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+  - name: config-volume-g3auto
+    readOnly: true
+    mountPath: /mds/.env
+    subPath: metadata.env
 # -- (map) Resource limits for the init container.
 initResources:
   # -- (map) The maximum amount of resources that the container is allowed to use
@@ -300,7 +310,8 @@ command: ["/bin/sh"]
 args:
   - "-c"
   - |
-    /env/bin/alembic upgrade head
+    # Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility
+    poetry run alembic upgrade head || /env/bin/alembic upgrade head
 
 # Service and Pod
 serviceAnnotations:

helm/portal/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.18
+version: 0.1.19
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/requestor/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.12
+version: 0.1.13
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/requestor/README.md

@@ -1,6 +1,6 @@
 # requestor
 
-![Version: 0.1.12](https://img.shields.io/badge/Version-0.1.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.13](https://img.shields.io/badge/Version-0.1.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Requestor Service
 
@@ -23,7 +23,7 @@ A Helm chart for gen3 Requestor Service
 | affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["requestor"]` | Value for the match expression key. |
 | affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. |
 | arboristUrl | string | `"http://arborist-service"` | Arborist service URL. |
-| args | list | `["-c","/env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
+| args | list | `["-c","# Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility\npoetry run alembic upgrade head || /env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. |
 | automountServiceAccountToken | bool | `false` | Automount the default service account token |
 | autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas |
 | autoscaling.enabled | bool | `false` | Whether autoscaling is enabled |

helm/requestor/values.yaml

@@ -190,6 +190,11 @@ volumeMounts:
   #   readOnly: true
   #   mountPath: "/src/requestor-config.yaml"
   #   subPath: "requestor-config.yaml"
+  # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+  # - name: "config-volume"
+  #   readOnly: true
+  #   mountPath: "/requestor/requestor-config.yaml"
+  #   subPath: "requestor-config.yaml"
 
 # -- (map) Resource requests and limits for the containers in the pod
 resources:
@@ -223,7 +228,8 @@ command: ["/bin/sh"]
 args:
   - "-c"
   - |
-    /env/bin/alembic upgrade head
+    # Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility
+    poetry run alembic upgrade head || /env/bin/alembic upgrade head
 
 # Service and Pod
 # -- (map) Kubernetes service information.

helm/wts/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.14
+version: 0.1.15
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/wts/README.md

@@ -1,6 +1,6 @@
 # wts
 
-![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.15](https://img.shields.io/badge/Version-0.1.15-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 workspace token service
 

helm/wts/templates/deployment.yaml

@@ -185,7 +185,8 @@ spec:
             if hash alembic 2>/dev/null; then
               echo "Running DB migration"
               cd /wts
-              alembic upgrade head
+              # Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility 
+              poetry run alembic upgrade head || alembic upgrade head
             else
               echo "Alembic not installed - not running DB migration"
             fi