forked from rust-lang/rust
-
Notifications
You must be signed in to change notification settings - Fork 0
336 lines (305 loc) · 15.6 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
# This file defines our primary CI workflow that runs on pull requests
# and also on pushes to special branches (auto, try).
#
# The actual definition of the executed jobs is calculated by a Python
# script located at src/ci/github-actions/calculate-job-matrix.py, which
# uses job definition data from src/ci/github-actions/jobs.yml.
# You should primarily modify the `jobs.yml` file if you want to modify
# what jobs are executed in CI.
name: CI
on:
push:
# branches:
# - auto
# - try
# - try-perf
# - automation/bors/try
pull_request:
branches:
- "**"
permissions:
contents: read
packages: write
defaults:
run:
# On Linux, macOS, and Windows, use the system-provided bash as the default
# shell. (This should only make a difference on Windows, where the default
# shell is PowerShell.)
shell: bash
concurrency:
# For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
# We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
# are all triggered on the same branch, but which should be able to run concurrently.
group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
cancel-in-progress: true
env:
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
# This will be empty in PR jobs.
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
jobs:
# The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
# It calculates which jobs should be executed, based on the data of the ${{ github }} context.
# If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
# calculate_matrix:
# name: Calculate job matrix
# runs-on: ubuntu-latest
# outputs:
# jobs: ${{ steps.jobs.outputs.jobs }}
# run_type: ${{ steps.jobs.outputs.run_type }}
# steps:
# - name: Checkout the source code
# uses: actions/checkout@v4
# - name: Calculate the CI job matrix
# env:
# COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
# run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
# id: jobs
job:
name: ${{ matrix.name }}
# needs: [ calculate_matrix ]
runs-on: "${{ matrix.os }}"
defaults:
run:
shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
timeout-minutes: 600
env:
CI_JOB_NAME: ${{ matrix.image }}
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SCCACHE_BUCKET: rust-lang-ci-sccache2
CACHE_DOMAIN: ci-caches.rust-lang.org
continue-on-error: ${{ matrix.continue_on_error || false }}
strategy:
matrix:
# Check the `calculate_matrix` job to see how is the matrix defined.
include:
- image: x86_64-msvc-ext1
name: x86_64-msvc-ext1
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
- image: x86_64-msvc-ext2
name: x86_64-msvc-ext2
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
- image: x86_64-msvc-ext3
name: x86_64-msvc-ext3
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
- image: x86_64-msvc-ext4
name: x86_64-msvc-ext4
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
- image: x86_64-msvc-ext5
name: x86_64-msvc-ext5
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
- image: x86_64-msvc-ext6
name: x86_64-msvc-ext6
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
- image: x86_64-msvc-ext7
name: x86_64-msvc-ext7
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
- image: x86_64-msvc-ext8
name: x86_64-msvc-ext8
os: windows-2022
env:
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
DEPLOY_BUCKET: rust-lang-ci2
DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
HOST_TARGET: x86_64-pc-windows-msvc
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
steps:
- if: contains(matrix.os, 'windows')
uses: msys2/[email protected]
with:
# i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
# don't try to download updates for already installed packages
update: false
# don't try to use the msys that comes built-in to the github runner,
# so we can control what is installed (i.e. not python)
release: true
# Inherit the full path from the Windows environment, with MSYS2's */bin/
# dirs placed in front. This lets us run Windows-native Python etc.
path-type: inherit
install: >
make
- name: disable git crlf conversion
run: git config --global core.autocrlf false
- name: checkout the source code
uses: actions/checkout@v4
with:
fetch-depth: 2
# Rust Log Analyzer can't currently detect the PR number of a GitHub
# Actions build on its own, so a hint in the log message is needed to
# point it in the right direction.
# - name: configure the PR in which the error message will be posted
# run: echo "[CI_PR_NUMBER=$num]"
# env:
# num: ${{ github.event.number }}
# if: needs.calculate_matrix.outputs.run_type == 'pr'
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
# Since it's not possible to merge `${{ matrix.env }}` with the other
# variables in `job.<name>.env`, the variables defined in the matrix
# are passed to the `setup-environment.sh` script encoded in JSON,
# which then uses log commands to actually set them.
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
# - name: ensure the channel matches the target branch
# run: src/ci/scripts/verify-channel.sh
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
- name: install awscli
run: src/ci/scripts/install-awscli.sh
- name: install sccache
run: src/ci/scripts/install-sccache.sh
- name: select Xcode
run: src/ci/scripts/select-xcode.sh
- name: install clang
run: src/ci/scripts/install-clang.sh
- name: install tidy
run: src/ci/scripts/install-tidy.sh
- name: install WIX
run: src/ci/scripts/install-wix.sh
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
- name: install ninja
run: src/ci/scripts/install-ninja.sh
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
# Disable automatic line ending conversion (again). On Windows, when we're
# installing dependencies, something switches the git configuration directory or
# re-enables autocrlf. We've not tracked down the exact cause -- and there may
# be multiple -- but this should ensure submodules are checked out with the
# appropriate line endings.
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
- name: ensure backported commits are in upstream branches
run: src/ci/scripts/verify-backported-commits.sh
- name: ensure the stable version number is correct
run: src/ci/scripts/verify-stable-version-number.sh
- name: run the build
# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
run: src/ci/scripts/run-build-from-ci.sh 2>&1
env:
AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
# - name: create github artifacts
# run: src/ci/scripts/create-doc-artifacts.sh
# - name: upload artifacts to github
# uses: actions/upload-artifact@v4
# with:
# # name is set in previous step
# name: ${{ env.DOC_ARTIFACT_NAME }}
# path: obj/artifacts/doc
# if-no-files-found: ignore
# retention-days: 5
# - name: upload artifacts to S3
# run: src/ci/scripts/upload-artifacts.sh
# env:
# AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
# AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
# # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
# # builders *should* have the AWS credentials available. Still, explicitly
# # adding the condition is helpful as this way CI will not silently skip
# # deploying artifacts from a dist builder if the variables are misconfigured,
# # erroring about invalid credentials instead.
# if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
# This job isused to tell bors the final status of the build, as there is no practical way to detect
# when a workflow is successful listening to webhooks only in our current bors implementation (homu).
# outcome:
# name: bors build finished
# runs-on: ubuntu-latest
# needs: [ calculate_matrix, job ]
# # !cancelled() executes the job regardless of whether the previous jobs passed or failed
# if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
# steps:
# - name: checkout the source code
# uses: actions/checkout@v4
# with:
# fetch-depth: 2
# # Calculate the exit status of the whole CI workflow.
# # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
# # If a some dependent job has failed, this exits with 1.
# - name: calculate the correct exit status
# run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
# # Publish the toolstate if an auto build succeeds (just before push to master)
# - name: publish toolstate
# run: src/ci/publish_toolstate.sh
# shell: bash
# if: needs.calculate_matrix.outputs.run_type == 'auto'
# env:
# TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
# TOOLSTATE_PUBLISH: 1