Add chart install + test

.github/ct.yaml

@@ -7,4 +7,5 @@ chart-repos:
 helm-extra-args: --timeout 600s
 check-version-increment: true
 debug: false
-validate-maintainers: false
+validate-maintainers: false
+helm-dependency-extra-args: "--skip-refresh"

.github/workflows/lint_test.yaml

@@ -34,6 +34,18 @@ jobs:
 
       - name: Run chart-testing (lint)
         run: ct lint --config .github/ct.yaml
+
+  deploy-and-test-chart:
+    name: ${{ matrix.environments }} - gen3 data portal build
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create kind cluster
+        uses: helm/[email protected]
+        if: steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart install + testing
+        run: ct install --charts ./helm/gen3 --config .github/ct.yaml
 
   # deploy-charts-to-kind:
   #   name: ${{ matrix.environments }} - gen3 data portal build

helm/etl/Chart.yaml

@@ -22,5 +22,4 @@ version: 0.1.0
 # follow Semantic Versioning. They should reflect the version the application is using.
 appVersion: "master"
 
-dependencies:
-
+dependencies: []

helm/etl/values.yaml

@@ -15,7 +15,7 @@ image:
     pullPolicy: Always
     # -- (string) Overrides the image tag whose default is the chart appVersion.
     tag: "master"
-    
+
 
 # -- (list) Docker image pull secrets.
 imagePullSecrets: []
@@ -26,7 +26,7 @@ podAnnotations: {}
 
 # -- (map) Resource requests and limits for the containers in the pod
 resources:
-  tube: 
+  tube:
     # -- (map) The amount of resources that the container requests
     requests:
       # -- (string) The amount of CPU requested
@@ -56,88 +56,88 @@ resources:
 
 etlMapping:
   mappings:
-      - name: dev_case
-        doc_type: case
-        type: aggregator
-        root: case
-        props:
-          - name: submitter_id
-          - name: project_id
-          - name: disease_type
-          - name: primary_site
-        flatten_props:
-          - path: demographics
-            props:
-              - name: gender
-                value_mappings:
-                  - female: F
-                  - male: M
-              - name: race
-                value_mappings:
-                  - american indian or alaskan native: Indian
-              - name: ethnicity
-              - name: year_of_birth
-        aggregated_props:
-          - name: _samples_count
-            path: samples
-            fn: count
-          - name: _aliquots_count
-            path: samples.aliquots
-            fn: count
-          - name: _submitted_methylations_count
-            path: samples.aliquots.submitted_methylation_files
-            fn: count
-          - name: _submitted_copy_number_files_on_aliquots_count
-            path: samples.aliquots.submitted_copy_number_files
-            fn: count
-          - name: _read_groups_count
-            path: samples.aliquots.read_groups
-            fn: count
-          - name: _submitted_aligned_reads_count
-            path: samples.aliquots.read_groups.submitted_aligned_reads_files
-            fn: count
-          - name: _submitted_unaligned_reads_count
-            path: samples.aliquots.read_groups.submitted_unaligned_reads_files
-            fn: count
-          - name: _submitted_copy_number_files_on_read_groups_count
-            path: samples.aliquots.read_groups.submitted_copy_number_files
-            fn: count
-          - name: _submitted_somatic_mutations_count
-            path: samples.aliquots.read_groups.submitted_somatic_mutations
-            fn: count
-        joining_props:
-          - index: file
-            join_on: _case_id
-            props:
-              - name: data_format
-                src: data_format
-                fn: set
-              - name: data_type
-                src: data_type
-                fn: set
-              - name: _file_id
-                src: _file_id
-                fn: set
-      - name: dev_file
-        doc_type: file
-        type: collector
-        root: None
-        category: data_file
-        props:
-          - name: object_id
-          - name: md5sum
-          - name: file_name
-          - name: file_size
-          - name: data_format
-          - name: data_type
-          - name: state
-        injecting_props:
-          case:
-            props:
-              - name: _case_id
-                src: id
-                fn: set
-              - name: project_id
-        target_nodes:
-          - name: slide_image
-            path: slides.samples.cases
+    - name: dev_case
+      doc_type: case
+      type: aggregator
+      root: case
+      props:
+        - name: submitter_id
+        - name: project_id
+        - name: disease_type
+        - name: primary_site
+      flatten_props:
+        - path: demographics
+          props:
+            - name: gender
+              value_mappings:
+                - female: F
+                - male: M
+            - name: race
+              value_mappings:
+                - american indian or alaskan native: Indian
+            - name: ethnicity
+            - name: year_of_birth
+      aggregated_props:
+        - name: _samples_count
+          path: samples
+          fn: count
+        - name: _aliquots_count
+          path: samples.aliquots
+          fn: count
+        - name: _submitted_methylations_count
+          path: samples.aliquots.submitted_methylation_files
+          fn: count
+        - name: _submitted_copy_number_files_on_aliquots_count
+          path: samples.aliquots.submitted_copy_number_files
+          fn: count
+        - name: _read_groups_count
+          path: samples.aliquots.read_groups
+          fn: count
+        - name: _submitted_aligned_reads_count
+          path: samples.aliquots.read_groups.submitted_aligned_reads_files
+          fn: count
+        - name: _submitted_unaligned_reads_count
+          path: samples.aliquots.read_groups.submitted_unaligned_reads_files
+          fn: count
+        - name: _submitted_copy_number_files_on_read_groups_count
+          path: samples.aliquots.read_groups.submitted_copy_number_files
+          fn: count
+        - name: _submitted_somatic_mutations_count
+          path: samples.aliquots.read_groups.submitted_somatic_mutations
+          fn: count
+      joining_props:
+        - index: file
+          join_on: _case_id
+          props:
+            - name: data_format
+              src: data_format
+              fn: set
+            - name: data_type
+              src: data_type
+              fn: set
+            - name: _file_id
+              src: _file_id
+              fn: set
+    - name: dev_file
+      doc_type: file
+      type: collector
+      root: None
+      category: data_file
+      props:
+        - name: object_id
+        - name: md5sum
+        - name: file_name
+        - name: file_size
+        - name: data_format
+        - name: data_type
+        - name: state
+      injecting_props:
+        case:
+          props:
+            - name: _case_id
+              src: id
+              fn: set
+            - name: project_id
+      target_nodes:
+        - name: slide_image
+          path: slides.samples.cases

helm/gen3/README.md

@@ -68,7 +68,7 @@ Helm chart to deploy Gen3 Data Commons
 | aws-es-proxy.secrets.awsAccessKeyId | str | `""` | AWS access key ID for aws-es-proxy |
 | aws-es-proxy.secrets.awsSecretAccessKey | str | `""` | AWS secret access key for aws-es-proxy |
 | elasticsearch.clusterName | string | `"gen3-elasticsearch"` |  |
-| elasticsearch.esConfig."elasticsearch.yml" | string | `"# Here we can add elasticsearch config"` |  |
+| elasticsearch.esConfig."elasticsearch.yml" | string | `"# Here we can add elasticsearch config\n"` |  |
 | elasticsearch.maxUnavailable | int | `0` |  |
 | elasticsearch.replicas | int | `1` |  |
 | elasticsearch.singleNode | bool | `true` |  |

helm/gen3/ci/portal-values.yaml

@@ -0,0 +1,9 @@
+portal:
+  image:
+    repository: quay.io/cdis/data-portal-prebuilt
+    tag: "toxcommons.com-master"
+
+  resources:
+    requests:
+      cpu: "0.2"
+      memory: 100Mi

helm/gen3/values.yaml

@@ -397,4 +397,4 @@ elasticsearch:
   replicas: 1
   esConfig:
     elasticsearch.yml: |
-      # Here we can add elasticsearch config
+      # Here we can add elasticsearch config

helm/wts/templates/wts-oidc.yaml

@@ -100,10 +100,11 @@ spec:
           args: 
             - "-c"
             - |
+              echo "waiting for /shared/client_id"
               while [ ! -e /shared/client_id ]
               do
-                echo "waiting for /shared/client_id"
-                sleep 30
+                echo "..."
+                sleep 5
               done
               echo "Updating k8s secret wts-oidc-client"
               CLIENT_ID=$(cat /shared/client_id | base64)