-
Notifications
You must be signed in to change notification settings - Fork 138
264 lines (258 loc) · 10.8 KB
/
test-integration-main.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
# TODO: Move the PR trigger to its own workflow and use this workflow as template only.
name: "Test - Integration - Main"
on:
pull_request_target:
paths:
- '.github/workflows/test-integration-main.yaml'
- '.tool-versions'
- 'charts/**'
types: [opened, closed, reopened, synchronize]
workflow_call:
inputs:
persistent:
description: |
Keep test deployment after the workflow is done.
NOTE: All persistent deployments will be deleted frequently to save costs!
required: false
default: false
type: boolean
platforms:
default: gke
type: string
identifier:
description: The unique identifier of used in the deployment hostname.
required: true
type: string
extra-values:
description: Pass extra values to the Helm chart.
required: false
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.identifier || github.ref }}
cancel-in-progress: true
env:
TEST_PERSISTENT: ${{ inputs.persistent || contains(github.event.*.labels.*.name, 'test-persistent') }}
TEST_HOSTNAME_BASE: ci.distro.ultrawombat.com
TEST_SCENARIOS_DIR: charts/camunda-platform/test/integration/scenarios
# Docker Hub auth to avoid image pull rate limit.
TEST_CREATE_DOCKER_LOGIN_SECRET: "TRUE"
TEST_DOCKER_USERNAME: ${{ secrets.DISTRO_CI_DOCKER_USERNAME_DOCKERHUB }}
TEST_DOCKER_PASSWORD: ${{ secrets.DISTRO_CI_DOCKER_PASSWORD_DOCKERHUB }}
# Camunda registry auth to access WebModeler Docker image since it's not public.
TEST_DOCKER_USERNAME_CAMUNDA_CLOUD: ${{ secrets.DISTRO_CI_DOCKER_USERNAME_CAMUNDA }}
TEST_DOCKER_PASSWORD_CAMUNDA_CLOUD: ${{ secrets.DISTRO_CI_DOCKER_PASSWORD_CAMUNDA }}
jobs:
test:
name: ${{ matrix.distro.name }} - ${{ matrix.scenario.name }}
if: github.event.action != 'closed'
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
deployments: write
strategy:
fail-fast: false
matrix:
distro:
- name: Kubernetes 1.24
type: kubernetes
platform: gke
secret:
cluster-name: DISTRO_CI_GCP_GKE_CLUSTER_NAME
cluster-location: DISTRO_CI_GCP_GKE_CLUSTER_LOCATION
workload-identity-provider: DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER
service-account: DISTRO_CI_GCP_SERVICE_ACCOUNT
if: ${{ github.event.pull_request.number || contains(inputs.platforms, 'gke') }}
- name: OpenShift 4.13
type: openshift
version: 4.13
platform: rosa
if: ${{ github.event.pull_request.number || contains(inputs.platforms, 'rosa') }}
scenario:
- name: Chart Setup
desc: Setup chart in production-like setup with Ingress and TLS.
flow: install
- name: Chart Upgrade
desc: Upgrade chart from the latest released version to the current branch.
flow: upgrade
exclude:
- distro:
if: false
env:
TEST_CLUSTER_TYPE: ${{ matrix.distro.type }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
# This is needed to load repo GH composite actions if the workflow triggered by workflow_call.
repository: camunda/camunda-platform-helm
# TODO: Later, find a way to abstract the auth for different platforms.
- name: Authenticate to GKE
if: matrix.distro.platform == 'gke'
uses: ./.github/actions/gke-login
with:
cluster-name: ${{ secrets[matrix.distro.secret.cluster-name] }}
cluster-location: ${{ secrets[matrix.distro.secret.cluster-location] }}
workload-identity-provider: ${{ secrets[matrix.distro.secret.workload-identity-provider] }}
service-account: ${{ secrets[matrix.distro.secret.service-account] }}
- name: Set OpenShift authentication vars
if: matrix.distro.type == 'openshift'
run: |
OPENSHIFT_CLUSTER_VERSION="$(echo ${{ matrix.distro.version }} | tr -d '.')"
echo "OPENSHIFT_CLUSTER_URL=OPENSHIFT_CLUSTER_URL_${OPENSHIFT_CLUSTER_VERSION}" >> $GITHUB_ENV
echo "OPENSHIFT_CLUSTER_USERNAME=OPENSHIFT_CLUSTER_USERNAME_${OPENSHIFT_CLUSTER_VERSION}" >> $GITHUB_ENV
echo "OPENSHIFT_CLUSTER_PASSWORD=OPENSHIFT_CLUSTER_PASSWORD_${OPENSHIFT_CLUSTER_VERSION}" >> $GITHUB_ENV
- name: Authenticate to OpenShift
if: matrix.distro.platform == 'rosa'
uses: redhat-actions/oc-login@v1
with:
openshift_server_url: ${{ secrets[env.OPENSHIFT_CLUSTER_URL] }}
openshift_username: ${{ secrets[env.OPENSHIFT_CLUSTER_USERNAME] }}
openshift_password: ${{ secrets[env.OPENSHIFT_CLUSTER_PASSWORD] }}
- name: Set workflow vars
id: vars
uses: ./.github/actions/workflow-vars
with:
persistent: ${{ env.TEST_PERSISTENT }}
setup-flow: ${{ matrix.scenario.flow }}
platform: ${{ matrix.distro.platform }}
identifier-base: ${{ github.event.pull_request.number || inputs.identifier }}
ingress-hostname-base: ${{ env.TEST_HOSTNAME_BASE }}
- name: Install env dependencies
uses: asdf-vm/actions/install@v3
- name: Add Helm repos and dependencies
run: |
make helm.repos-add
make helm.dependency-update
- name: Create test namespace
run: |
echo $TEST_NAMESPACE
kubectl delete ns --ignore-not-found=true \
-l "github-id=${{ steps.vars.outputs.identifier }},test-flow=${{ matrix.scenario.flow }},test-persistent=true"
kubectl create ns $TEST_NAMESPACE
kubectl label ns $TEST_NAMESPACE github-run-id=$GITHUB_WORKFLOW_RUN_ID
kubectl label ns $TEST_NAMESPACE github-job-id=$GITHUB_WORKFLOW_JOB_ID
kubectl label ns $TEST_NAMESPACE github-id=${{ steps.vars.outputs.identifier }}
kubectl label ns $TEST_NAMESPACE test-flow=${{ matrix.scenario.flow }}
kubectl label ns $TEST_NAMESPACE test-persistent=${{ env.TEST_PERSISTENT }}
- name: Copy PRs wildcard certificate
run: |
kubectl apply -n $TEST_NAMESPACE -f .github/config/external-secret.yaml
- name: Start GitHub deployment
uses: bobheadxi/deployments@v1
id: deployment
with:
step: start
token: ${{ secrets.GITHUB_TOKEN }}
env: ${{ steps.vars.outputs.identifier }}
ref: ${{ github.event.pull_request.head.sha }}
- name: Pre setup
timeout-minutes: 5
env:
TEST_CHART_FLOW: ${{ matrix.scenario.flow }}
TEST_INGRESS_HOST: ${{ steps.vars.outputs.ingress-host }}
run: |
task -d $TEST_SCENARIOS_DIR/chart-full-setup setup.pre
echo "Extra values from workflow:"
echo "${{ inputs.extra-values }}" > /tmp/extra-values-file.yaml
cat /tmp/extra-values-file.yaml
- name: 🌟 Setup Camunda chart 🌟
env:
TEST_CHART_FLOW: ${{ matrix.scenario.flow }}
TEST_HELM_EXTRA_ARGS: >-
--set global.ingress.host=${{ steps.vars.outputs.ingress-host }}
--values /tmp/extra-values-file.yaml
TEST_PROMETHEUS_HELM_EXTRA_ARGS: "--set host=${{ steps.ingress.outputs.host }}"
run: |
task -d $TEST_SCENARIOS_DIR/chart-full-setup setup.exec
- name: Post setup
timeout-minutes: 5
run: |
task -d $TEST_SCENARIOS_DIR/chart-full-setup setup.post
- name: Pre Upgrade
if: matrix.scenario.flow == 'upgrade'
run: |
task -d $TEST_SCENARIOS_DIR/chart-full-setup upgrade.pre
- name: 🌟 Upgrade Camunda chart 🌟
if: matrix.scenario.flow == 'upgrade'
env:
TEST_HELM_EXTRA_ARGS: >-
--set global.ingress.host=${{ steps.vars.outputs.ingress-host }}
--values /tmp/extra-values-file.yaml
run: |
task -d $TEST_SCENARIOS_DIR/chart-full-setup upgrade.exec
- name: Update GitHub deployment status
uses: bobheadxi/deployments@v1
with:
step: finish
token: ${{ secrets.GITHUB_TOKEN }}
status: ${{ job.status }}
env_url: https://${{ steps.vars.outputs.ingress-host }}
deployment_id: ${{ steps.deployment.outputs.deployment_id }}
env: ${{ steps.deployment.outputs.env }}
ref: ${{ github.event.pull_request.head.sha }}
- name: ⭐️ Run Preflight TestSuite ⭐️
timeout-minutes: 10
run: |
task -d $TEST_SCENARIOS_DIR/chart-full-setup test.preflight
- name: ⭐️ Run Core TestSuite ⭐️
timeout-minutes: 20
run: |
task -d $TEST_SCENARIOS_DIR/chart-full-setup test.core
- name: Get failed Pods info
if: failure()
uses: ./.github/actions/failed-pods-info
# TODO: Use "step: delete-env" to delete the env when the permission issue is fixed.
# Even using GH app token with deployment write access doesn't work.
# https://github.com/bobheadxi/deployments/issues/145
- name: Cleanup GitHub deployment
if: always() && (env.TEST_PERSISTENT == 'false' || matrix.distro.type != 'kubernetes')
uses: bobheadxi/deployments@v1
with:
step: deactivate-env
token: ${{ secrets.GITHUB_TOKEN }}
env: ${{ steps.deployment.outputs.env }}
ref: ${{ github.event.pull_request.head.sha }}
- name: Cleanup test namespace
if: always() && (env.TEST_PERSISTENT == 'false' || matrix.distro.type != 'kubernetes')
run: |
kubectl delete ns --ignore-not-found=true \
-l github-run-id=$GITHUB_WORKFLOW_RUN_ID \
-l github-job-id=$GITHUB_WORKFLOW_JOB_ID
clean:
name: Clean up persistent resources
if: github.event.action == 'closed'
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
deployments: write
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
- name: Set PR vars
id: vars
uses: ./.github/actions/workflow-vars
with:
persistent: ${{ env.TEST_PERSISTENT }}
platform: gke
identifier-base: ${{ github.event.pull_request.number || inputs.identifier }}
# Persistent resources are deployed only on GKE.
- name: Authenticate to GKE
uses: ./.github/actions/gke-login
with:
cluster-name: ${{ secrets.DISTRO_CI_GCP_GKE_CLUSTER_NAME }}
cluster-location: ${{ secrets.DISTRO_CI_GCP_GKE_CLUSTER_LOCATION }}
workload-identity-provider: ${{ secrets.DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER }}
service-account: ${{ secrets.DISTRO_CI_GCP_SERVICE_ACCOUNT }}
- name: Cleanup GitHub deployment
if: always()
uses: bobheadxi/deployments@v1
with:
step: deactivate-env
token: ${{ secrets.GITHUB_TOKEN }}
env: ${{ steps.vars.outputs.identifier }}
ref: ${{ github.event.pull_request.head.sha }}
- name: Cleanup test namespace
if: always()
run: |
kubectl delete ns --ignore-not-found=true \
-l "github-id=${{ steps.vars.outputs.identifier }},test-persistent=true"