forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
162 lines (151 loc) · 6.19 KB
/
gke-kube-test-command.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
name: GKE Kube Acceptance Test
on:
workflow_dispatch:
inputs:
repo:
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
required: false
default: "airbytehq/airbyte"
gitref:
description: "The git ref to check out from the specified repository."
required: false
default: master
comment-id:
description: "The comment-id of the slash command. Used to update the comment with the status."
required: false
jobs:
find_valid_pat:
name: "Find a PAT with room for actions"
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
pat: ${{ steps.variables.outputs.pat }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Check PAT rate limits
id: variables
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.AIRBYTEIO_PAT }} \
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
${{ secrets.SUPERTOPHER_PAT }} \
${{ secrets.DAVINCHIA_PAT }}
start-gke-kube-acceptance-test-runner:
timeout-minutes: 10
name: Start GKE Kube Acceptance Test EC2 Runner
runs-on: ubuntu-latest
needs: find_valid_pat
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ needs.find_valid_pat.outputs.pat }}
gke-kube-acceptance-test:
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
needs: start-gke-kube-acceptance-test-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
name: GKE Acceptance Tests (Kube)
timeout-minutes: 90
steps:
- name: Link comment to workflow run
if: github.event.inputs.comment-id
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :clock2: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
- name: Checkout Airbyte
uses: actions/checkout@v2
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
- uses: actions/setup-java@v1
with:
java-version: "17"
- uses: actions/setup-node@v2
with:
node-version: "lts/gallium"
- name: Fix EC-2 Runner
run: |
mkdir -p /home/runner
- name: Install socat (required for port forwarding)
run: |
sudo apt-get update
sudo apt-get install socat
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v0
with:
project_id: ${{ secrets.GKE_TEST_PROJECT_ID }}
service_account_key: ${{ secrets.GKE_TEST_SA_KEY }}
export_default_credentials: true
- name: Get GKE credentials
uses: google-github-actions/get-gke-credentials@main
with:
project_id: ${{ secrets.GKE_TEST_PROJECT_ID }}
credentials: ${{ secrets.GKE_TEST_SA_KEY }}
cluster_name: kube-acceptance-test-cluster
location: us-central1-c
- name: Install kubectl
run: |-
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
- name: Run End-to-End Acceptance Tests on GKE
id: gke-kube-test
env:
USER: root
HOME: /home/runner
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
ACTION_RUN_ID: ${{github.run_id}}
run: |
./tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh
- name: Add Success Comment
if: github.event.inputs.comment-id && success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
- name: Add Failure Comment
if: github.event.inputs.comment-id && !success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
# In case of self-hosted EC2 errors, remove this block.
stop-gke-kube-acceptance-test-runner:
name: Stop GKE Kube Acceptance Test EC2 Runner
timeout-minutes: 10
needs:
- start-gke-kube-acceptance-test-runner # required to get output from the start-runner job
- gke-kube-acceptance-test # required to wait when the main job is done
- find_valid_pat
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Stop EC2 runner
uses: supertopher/[email protected]
with:
mode: stop
github-token: ${{ needs.find_valid_pat.outputs.pat }}
label: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.ec2-instance-id }}