-
Notifications
You must be signed in to change notification settings - Fork 1.9k
183 lines (156 loc) · 5.87 KB
/
bench-turborepo.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
name: Benchmark Turborepo
on:
workflow_dispatch:
workflow_run:
# Make sure this matches the name of the workflow in ./github/workflows/turborepo-release.yml.
workflows: [Turborepo Release]
types:
- completed
push:
branches:
- main
paths:
- "cli/**"
- crates/turborepo*/**
# - "benchmark/**" (we don't need to run benchmarks when the benchmark changes, next push will capture it)
jobs:
benchmark:
name: Benchmark turbo run
timeout-minutes: 60
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-node
- name: Setup Turborepo Environment
uses: ./.github/actions/setup-turborepo-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
- name: Build
run: pnpm -F cli build:release
- name: Run benchmarks
run: pnpm -F @turbo/benchmark benchmark
- name: Save to Tinybird
run: |
curl \
-i \
-F "ndjson=@./packages/turbo-benchmark/tinybird.ndjson" \
-X POST \
-H 'Authorization: Bearer ${{ secrets.TINYBIRD_TOKEN }}' \
'https://api.us-east.tinybird.co/v0/datasources?format=ndjson&name=turbo_benchmarks&mode=append'
time-to-first-task:
name: Benchmark TTFT
timeout-minutes: 60
runs-on: ${{ matrix.os.runner }}
strategy:
fail-fast: false
matrix:
os:
- name: ubuntu
runner: ubuntu-latest
- name: macos
runner: macos-latest
- name: windows
runner: windows-latest
steps:
- uses: actions/checkout@v3
- name: Set filename for profile
id: filename
shell: bash
run: |
echo 'file_basename=${{ matrix.os.name }}' >> "$GITHUB_OUTPUT"
echo 'filename=${{ matrix.os.name }}.json' >> "$GITHUB_OUTPUT"
- name: Display filename
shell: bash
run: echo "${{ steps.filename.outputs.filename }}"
- name: Setup Turborepo Environment
uses: ./.github/actions/setup-turborepo-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
- name: Build Turborepo from source
run: pnpm -F cli build:release
- name: Run benchmarks
shell: bash
# ttft script will normalize filepath and place the profile in the benchmark directory.
run: pnpm -F @turbo/benchmark ttft "${{ steps.filename.outputs.filename }}"
- name: Upload Artifacts
uses: actions/upload-artifact@v4
with:
name: profiles-${{ matrix.os.name }} # This name will be the folder each file gets downloaded to
if-no-files-found: error
# cwd is root of the repository, so we need the benchmark/ prefixed path
path: |
packages/turbo-benchmark/profiles/${{ steps.filename.outputs.filename }}
packages/turbo-benchmark/profiles/${{ steps.filename.outputs.file_basename }}-ttft.json
# Send each of the profiles generated from the previous job to TinyBird
# We'll wait for all profiles to complete before sending.
send-to-tinybird:
name: Send to Tinybird
needs: [time-to-first-task]
runs-on: ubuntu-latest
env:
TINYBIRD_TOKEN: ${{secrets.TINYBIRD_TOKEN}}
steps:
- uses: actions/checkout@v3
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Download profiles
uses: actions/download-artifact@v4
with:
path: packages/turbo-benchmark/profiles/
pattern: profiles-*
merge-multiple: true
- name: Display TTFT Data
shell: bash
run: |
ls -al packages/turbo-benchmark/profiles
cat packages/turbo-benchmark/profiles/ubuntu-ttft.json
cat packages/turbo-benchmark/profiles/macos-ttft.json
cat packages/turbo-benchmark/profiles/windows-ttft.json
- name: Send data to TinyBird
shell: bash
run: |
cd packages/turbo-benchmark
node -r esbuild-register ./src/ttft/tinybird.ts profiles/ubuntu-ttft.json ${{github.run_id}}
node -r esbuild-register ./src/ttft/tinybird.ts profiles/macos-ttft.json ${{github.run_id}}
node -r esbuild-register ./src/ttft/tinybird.ts profiles/windows-ttft.json ${{github.run_id}}
send-to-slack:
name: Send to Slack
# Wait for send-to-tinybird so we can get aggregate data points
# before sending to slack.
needs: [send-to-tinybird]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Download profiles
uses: actions/download-artifact@v4
with:
path: packages/turbo-benchmark/profiles/
pattern: profiles-*
merge-multiple: true
- name: Display TTFT Data
shell: bash
run: |
ls -al packages/turbo-benchmark/profiles
cat packages/turbo-benchmark/profiles/ubuntu-ttft.json
cat packages/turbo-benchmark/profiles/macos-ttft.json
cat packages/turbo-benchmark/profiles/windows-ttft.json
# TODO: compare results to previous data and only post regressions
- name: Create Slack payload
shell: bash
env:
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_TOKEN }}
run: |
cd packages/turbo-benchmark
node -r esbuild-register ./src/ttft/slack.ts ${{github.run_id}}
- name: Debug Slack payload
shell: bash
run: cat packages/turbo-benchmark/slack-payload.json | jq
- name: Send payload to slack
uses: slackapi/[email protected]
with:
payload-file-path: "packages/turbo-benchmark/slack-payload.json"
env:
SLACK_WEBHOOK_URL: "${{ secrets.TURBOREPO_PERF_BENCHMARK_SLACK_WEBHOOK_URL }}"