From 894c8e6ee20de49f99c2236bfe4af877788e2298 Mon Sep 17 00:00:00 2001 From: Jawad Qureshi Date: Mon, 30 Oct 2023 18:29:47 -0500 Subject: [PATCH] Add chart install + test --- .github/ct.yaml | 3 +- .github/workflows/lint_test.yaml | 12 +++ helm/etl/Chart.yaml | 3 +- helm/etl/values.yaml | 174 +++++++++++++++---------------- helm/gen3/README.md | 2 +- helm/gen3/ci/portal-values.yaml | 9 ++ helm/gen3/values.yaml | 2 +- helm/wts/templates/wts-oidc.yaml | 5 +- 8 files changed, 116 insertions(+), 94 deletions(-) create mode 100644 helm/gen3/ci/portal-values.yaml diff --git a/.github/ct.yaml b/.github/ct.yaml index 531244f4..9126618a 100644 --- a/.github/ct.yaml +++ b/.github/ct.yaml @@ -7,4 +7,5 @@ chart-repos: helm-extra-args: --timeout 600s check-version-increment: true debug: false -validate-maintainers: false \ No newline at end of file +validate-maintainers: false +helm-dependency-extra-args: "--skip-refresh" \ No newline at end of file diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml index 60f85dca..89750db2 100644 --- a/.github/workflows/lint_test.yaml +++ b/.github/workflows/lint_test.yaml @@ -34,6 +34,18 @@ jobs: - name: Run chart-testing (lint) run: ct lint --config .github/ct.yaml + + deploy-and-test-chart: + name: ${{ matrix.environments }} - gen3 data portal build + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - name: Create kind cluster + uses: helm/kind-action@v1.4.0 + if: steps.list-changed.outputs.changed == 'true' + + - name: Run chart install + testing + run: ct install --charts ./helm/gen3 --config .github/ct.yaml # deploy-charts-to-kind: # name: ${{ matrix.environments }} - gen3 data portal build diff --git a/helm/etl/Chart.yaml b/helm/etl/Chart.yaml index e2479555..0f9e2fb9 100644 --- a/helm/etl/Chart.yaml +++ b/helm/etl/Chart.yaml @@ -22,5 +22,4 @@ version: 0.1.0 # follow Semantic Versioning. They should reflect the version the application is using. appVersion: "master" -dependencies: - +dependencies: [] diff --git a/helm/etl/values.yaml b/helm/etl/values.yaml index e292bfd5..4436f924 100644 --- a/helm/etl/values.yaml +++ b/helm/etl/values.yaml @@ -15,7 +15,7 @@ image: pullPolicy: Always # -- (string) Overrides the image tag whose default is the chart appVersion. tag: "master" - + # -- (list) Docker image pull secrets. imagePullSecrets: [] @@ -26,7 +26,7 @@ podAnnotations: {} # -- (map) Resource requests and limits for the containers in the pod resources: - tube: + tube: # -- (map) The amount of resources that the container requests requests: # -- (string) The amount of CPU requested @@ -56,88 +56,88 @@ resources: etlMapping: mappings: - - name: dev_case - doc_type: case - type: aggregator - root: case - props: - - name: submitter_id - - name: project_id - - name: disease_type - - name: primary_site - flatten_props: - - path: demographics - props: - - name: gender - value_mappings: - - female: F - - male: M - - name: race - value_mappings: - - american indian or alaskan native: Indian - - name: ethnicity - - name: year_of_birth - aggregated_props: - - name: _samples_count - path: samples - fn: count - - name: _aliquots_count - path: samples.aliquots - fn: count - - name: _submitted_methylations_count - path: samples.aliquots.submitted_methylation_files - fn: count - - name: _submitted_copy_number_files_on_aliquots_count - path: samples.aliquots.submitted_copy_number_files - fn: count - - name: _read_groups_count - path: samples.aliquots.read_groups - fn: count - - name: _submitted_aligned_reads_count - path: samples.aliquots.read_groups.submitted_aligned_reads_files - fn: count - - name: _submitted_unaligned_reads_count - path: samples.aliquots.read_groups.submitted_unaligned_reads_files - fn: count - - name: _submitted_copy_number_files_on_read_groups_count - path: samples.aliquots.read_groups.submitted_copy_number_files - fn: count - - name: _submitted_somatic_mutations_count - path: samples.aliquots.read_groups.submitted_somatic_mutations - fn: count - joining_props: - - index: file - join_on: _case_id - props: - - name: data_format - src: data_format - fn: set - - name: data_type - src: data_type - fn: set - - name: _file_id - src: _file_id - fn: set - - name: dev_file - doc_type: file - type: collector - root: None - category: data_file - props: - - name: object_id - - name: md5sum - - name: file_name - - name: file_size - - name: data_format - - name: data_type - - name: state - injecting_props: - case: - props: - - name: _case_id - src: id - fn: set - - name: project_id - target_nodes: - - name: slide_image - path: slides.samples.cases \ No newline at end of file + - name: dev_case + doc_type: case + type: aggregator + root: case + props: + - name: submitter_id + - name: project_id + - name: disease_type + - name: primary_site + flatten_props: + - path: demographics + props: + - name: gender + value_mappings: + - female: F + - male: M + - name: race + value_mappings: + - american indian or alaskan native: Indian + - name: ethnicity + - name: year_of_birth + aggregated_props: + - name: _samples_count + path: samples + fn: count + - name: _aliquots_count + path: samples.aliquots + fn: count + - name: _submitted_methylations_count + path: samples.aliquots.submitted_methylation_files + fn: count + - name: _submitted_copy_number_files_on_aliquots_count + path: samples.aliquots.submitted_copy_number_files + fn: count + - name: _read_groups_count + path: samples.aliquots.read_groups + fn: count + - name: _submitted_aligned_reads_count + path: samples.aliquots.read_groups.submitted_aligned_reads_files + fn: count + - name: _submitted_unaligned_reads_count + path: samples.aliquots.read_groups.submitted_unaligned_reads_files + fn: count + - name: _submitted_copy_number_files_on_read_groups_count + path: samples.aliquots.read_groups.submitted_copy_number_files + fn: count + - name: _submitted_somatic_mutations_count + path: samples.aliquots.read_groups.submitted_somatic_mutations + fn: count + joining_props: + - index: file + join_on: _case_id + props: + - name: data_format + src: data_format + fn: set + - name: data_type + src: data_type + fn: set + - name: _file_id + src: _file_id + fn: set + - name: dev_file + doc_type: file + type: collector + root: None + category: data_file + props: + - name: object_id + - name: md5sum + - name: file_name + - name: file_size + - name: data_format + - name: data_type + - name: state + injecting_props: + case: + props: + - name: _case_id + src: id + fn: set + - name: project_id + target_nodes: + - name: slide_image + path: slides.samples.cases diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 42ed29a5..38ebfc75 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -68,7 +68,7 @@ Helm chart to deploy Gen3 Data Commons | aws-es-proxy.secrets.awsAccessKeyId | str | `""` | AWS access key ID for aws-es-proxy | | aws-es-proxy.secrets.awsSecretAccessKey | str | `""` | AWS secret access key for aws-es-proxy | | elasticsearch.clusterName | string | `"gen3-elasticsearch"` | | -| elasticsearch.esConfig."elasticsearch.yml" | string | `"# Here we can add elasticsearch config"` | | +| elasticsearch.esConfig."elasticsearch.yml" | string | `"# Here we can add elasticsearch config\n"` | | | elasticsearch.maxUnavailable | int | `0` | | | elasticsearch.replicas | int | `1` | | | elasticsearch.singleNode | bool | `true` | | diff --git a/helm/gen3/ci/portal-values.yaml b/helm/gen3/ci/portal-values.yaml new file mode 100644 index 00000000..6cf7bb48 --- /dev/null +++ b/helm/gen3/ci/portal-values.yaml @@ -0,0 +1,9 @@ +portal: + image: + repository: quay.io/cdis/data-portal-prebuilt + tag: "toxcommons.com-master" + + resources: + requests: + cpu: "0.2" + memory: 100Mi \ No newline at end of file diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index ba7b0ad0..9fd862d1 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -397,4 +397,4 @@ elasticsearch: replicas: 1 esConfig: elasticsearch.yml: | - # Here we can add elasticsearch config \ No newline at end of file + # Here we can add elasticsearch config diff --git a/helm/wts/templates/wts-oidc.yaml b/helm/wts/templates/wts-oidc.yaml index d14c6e79..769a3c47 100644 --- a/helm/wts/templates/wts-oidc.yaml +++ b/helm/wts/templates/wts-oidc.yaml @@ -100,10 +100,11 @@ spec: args: - "-c" - | + echo "waiting for /shared/client_id" while [ ! -e /shared/client_id ] do - echo "waiting for /shared/client_id" - sleep 30 + echo "..." + sleep 5 done echo "Updating k8s secret wts-oidc-client" CLIENT_ID=$(cat /shared/client_id | base64)