-
Notifications
You must be signed in to change notification settings - Fork 282
180 lines (168 loc) · 6.35 KB
/
smoke.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
name: smoke
on:
pull_request_target:
types: [opened, synchronize, reopened]
branches:
- main
push:
branches:
- main
paths-ignore:
- docs/**
workflow_dispatch:
jobs:
check-label:
runs-on: ubuntu-22.04
outputs:
run_smoke_tests: ${{ steps.check.outputs.run_smoke_tests }}
steps:
- name: Check if PR author is a member of the organization or has the run-smoke label
id: check
run: |
case "${{ github.event_name }}" in
push)
# Run smoke tests for push to base repo
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
exit 0
;;
workflow_dispatch)
# Run smoke tests for manual runs against base branch
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
exit 0
;;
pull_request_target)
ORG="gptscript-ai"
AUTHOR="${{ github.event.pull_request.user.login }}"
# Check for org membership
MEMBERSHIP_RESPONSE_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.SMOKE_GH_TOKEN }}" \
"https://api.github.com/orgs/$ORG/members/$AUTHOR")
if [ "$MEMBERSHIP_RESPONSE_CODE" -eq 204 ]; then
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
exit 0
fi
# Check for "run-smoke" label
LABELS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/${{ github.repository_owner }}/${{ github.event.repository.name }}/issues/${{ github.event.pull_request.number }}/labels" | jq -r '.[].name')
if echo "$LABELS" | grep -q "run-smoke"; then
# Run smoke tests for PR with the "run-smoke" label
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
exit 0
fi
;;
esac
echo "run_smoke_tests=false" >> $GITHUB_OUTPUT
gpt-4o-2024-05-13:
needs: check-label
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
runs-on: ubuntu-22.04
steps:
- name: Checkout base repository
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Checkout PR code if running for a PR
if: ${{ github.event_name == 'pull_request_target' }}
uses: actions/checkout@v4
with:
fetch-depth: 1
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
- uses: actions/setup-go@v5
with:
cache: false
go-version: "1.21"
- env:
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-05-13
name: Run smoke test for gpt-4o-2024-05-13
run: |
echo "Running smoke test for model gpt-4o-2024-05-13"
export PATH="$(pwd)/bin:${PATH}"
make smoke
gpt-4-turbo-2024-04-09:
needs: check-label
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
runs-on: ubuntu-22.04
steps:
- name: Checkout base repository
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Checkout PR code if running for a PR
if: ${{ github.event_name == 'pull_request_target' }}
uses: actions/checkout@v4
with:
fetch-depth: 1
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
- uses: actions/setup-go@v5
with:
cache: false
go-version: "1.21"
- env:
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
GPTSCRIPT_DEFAULT_MODEL: gpt-4-turbo-2024-04-09
name: Run smoke test for gpt-4-turbo-2024-04-09
run: |
echo "Running smoke test for model gpt-4-turbo-2024-04-09"
export PATH="$(pwd)/bin:${PATH}"
make smoke
claude-3-opus-20240229:
needs: check-label
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
runs-on: ubuntu-22.04
steps:
- name: Checkout base repository
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Checkout PR code if running for a PR
if: ${{ github.event_name == 'pull_request_target' }}
uses: actions/checkout@v4
with:
fetch-depth: 1
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
- uses: actions/setup-go@v5
with:
cache: false
go-version: "1.21"
- env:
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
GPTSCRIPT_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider
ANTHROPIC_API_KEY: ${{ secrets.SMOKE_ANTHROPIC_API_KEY }}
GPTSCRIPT_CREDENTIAL_OVERRIDE: "github.com/gptscript-ai/claude3-anthropic-provider/credential:ANTHROPIC_API_KEY"
name: Run smoke test for claude-3-opus-20240229
run: |
echo "Running smoke test for model claude-3-opus-20240229"
export PATH="$(pwd)/bin:${PATH}"
make smoke
mistral-large-2402:
needs: check-label
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
runs-on: ubuntu-22.04
steps:
- name: Checkout base repository
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Checkout PR code if running for a PR
if: ${{ github.event_name == 'pull_request_target' }}
uses: actions/checkout@v4
with:
fetch-depth: 1
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
- uses: actions/setup-go@v5
with:
cache: false
go-version: "1.21"
- env:
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
GPTSCRIPT_DEFAULT_MODEL: mistral-large-2402 from https://api.mistral.ai/v1
GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY: ${{ secrets.SMOKE_GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY }}
name: Run smoke test for mistral-large-2402
run: |
echo "Running smoke test for model mistral-large-2402"
export PATH="$(pwd)/bin:${PATH}"
make smoke