-
Notifications
You must be signed in to change notification settings - Fork 0
134 lines (102 loc) · 5.96 KB
/
e2e_tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
name: End-to-End tests
#on: [push, pull_request]
on: [push]
#on:
# pull_request_target:
# types: [labeled]
jobs:
test:
runs-on: ubuntu-latest
environment: "GitHub Actions 1"
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ASSISTANTS_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ASSISTANTS_API_TYPE: ${{ secrets.ASSISTANTS_API_TYPE }}
ASSISTANTS_ID: ${{ secrets.ASSISTANTS_ID }}
ASSISTANTS_BASE_URL: ${{ secrets.ASSISTANTS_BASE_URL }}
ASSISTANTS_MODEL: ${{ secrets.ASSISTANTS_MODEL }}
ASSISTANTS_BOT_NAME: ${{ secrets.ASSISTANTS_BOT_NAME }}
POSTGRES_DATA_HOST: ${{ secrets.POSTGRES_DATA_HOST }}
POSTGRES_DATA_PORT: ${{ secrets.POSTGRES_DATA_PORT }}
POSTGRES_DATA_DB: ${{ secrets.POSTGRES_DATA_DB }}
POSTGRES_DATA_USER: ${{ secrets.POSTGRES_DATA_USER }}
POSTGRES_DATA_PASSWORD: ${{ secrets.POSTGRES_DATA_PASSWORD }}
DATA_DB_CONN_STRING: ${{ secrets.DATA_DB_CONN_STRING }}
POSTGRES_RECIPE_HOST: ${{ secrets.POSTGRES_RECIPE_HOST }}
POSTGRES_RECIPE_PORT: ${{ secrets.POSTGRES_RECIPE_PORT }}
POSTGRES_RECIPE_DB: ${{ secrets.POSTGRES_RECIPE_DB }}
POSTGRES_RECIPE_USER: ${{ secrets.POSTGRES_RECIPE_USER }}
POSTGRES_RECIPE_PASSWORD: ${{ secrets.POSTGRES_RECIPE_PASSWORD }}
RECIPE_DB_CONN_STRING: "postgresql://${{ secrets.POSTGRES_RECIPE_USER }}:${{ secrets.POSTGRES_RECIPE_PASSWORD }}@${{ secrets.POSTGRES_RECIPE_HOST }}:${{ secrets.POSTGRES_RECIPE_PORT }}/${{ secrets.POSTGRES_RECIPE_DB }}"
RECIPES_OPENAI_API_TYPE: ${{ secrets.RECIPES_OPENAI_API_TYPE }}
RECIPES_OPENAI_API_KEY: ${{ secrets.RECIPES_OPENAI_API_KEY }}
RECIPES_MODEL: ${{ secrets.RECIPES_MODEL }}
RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME: ${{ secrets.RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME }}
RECIPES_MEMORY_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_MEMORY_SIMILARITY_CUTOFF }}
RECIPES_RECIPE_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_RECIPE_SIMILARITY_CUTOFF }}
RECIPES_HELPER_FUNCTION_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_HELPER_FUNCTION_SIMILARITY_CUTOFF }}
RECIPES_MODEL_TEMP: ${{ secrets.RECIPES_MODEL_TEMP }}
RECIPES_MODEL_MAX_TOKENS: ${{ secrets.RECIPES_MODEL_MAX_TOKENS }}
IMAGE_HOST: ${{ secrets.IMAGE_HOST }}
RECIPE_SERVER_API: ${{ secrets.RECIPE_SERVER_API_FROM_GH_HOST }}
CHAT_URL: ${{ secrets.CHAT_URL }}
CHAINLIT_AUTH_SECRET: ${{ secrets.CHAINLIT_AUTH_SECRET }}
USER_LOGIN: ${{ secrets.USER_LOGIN }}
USER_PASSWORD: ${{ secrets.USER_PASSWORD }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Expose GitHub Runtime
uses: crazy-max/ghaction-github-runtime@v2
- name: Spin up Docker containers
run: |
env > .env
echo "Installing demo data ..."
# Get demo data
pip3 install gdown==5.2.0
cd data && python3 download_demo_data.py && cd ..
# TODO this should be enhanced to use a buildx bake to leverage layer caching for faster builds, or push to repo and simply have a pull for the run
# TODO docker-compose files should be refactored to use scopes instead of different versions for each environment
echo "Starting docker containers for dbs and server ..."
docker-compose pull
docker-compose up -d --build
# TODO: For some reason, maybe buildkit, in Github docker compose builds the image differently, and it doesn't work. Individual image build works.
docker build --build-arg OPENAI_API_KEY=$OPENAI_API_KEY --build-arg CHAT_URL=$CHAT_URL --build-arg OPENAI_API_ENDPOINT=$OPENAI_API_ENDPOINT --no-cache -t promptflow -f ./flows/chainlit-ui-evaluation/Dockerfile .
docker run --env RECIPES_MODEL_MAX_TOKENS=${RECIPES_MODEL_MAX_TOKENS} --env RECIPES_MODEL_TEMP=${RECIPES_MODEL_TEMP} --env RECIPES_OPENAI_API_TYPE=${ASSISTANTS_API_TYPE} --env RECIPES_OPENAI_API_KEY=${ASSISTANTS_API_KEY} --env RECIPES_MODEL=${RECIPES_MODEL} --env RECIPES_BASE_URL=${RECIPES_BASE_URL} --env USER_LOGIN=${USER_LOGIN} --env USER_PASSWORD=${USER_PASSWORD} --env CHAT_URL=${CHAT_URL} --network=data-recipes-ai_default -d --name promptflow promptflow
- name: Check logs
run: |
docker ps
echo "logs datadb ..."
docker compose logs datadb
echo "logs promptflow ..."
docker logs promptflow
echo "logs chat ..."
docker compose logs chat
echo "logs server ..."
docker compose logs server
docker ps
sleep 10
# Debugging GitHUb actions interactively, by connecting to the runner ...
# Get ssh connection details for runner
# See here https://github.com/marketplace/actions/debugging-with-ssh
# Basically, uncomment this, then get connection string in actions output, then connect with
#
# ssh -i <YOUR GITHUB SSH KEY> <CONN STRING ON ACTIONS>
#
#- name: DEBUG - Setup upterm session
# uses: lhotari/action-upterm@v1
# - name: DEBUG - Run Selenium outside of promptflow
# run: |
# docker exec promptflow python call_assistant.py
- name: Run tests
run: |
env > .env
docker exec promptflow pf run create --flow . --data ./data.jsonl --stream --column-mapping query='${data.query}' context='${data.context}' chat_history='${data.chat_history}' --name base_run
- name: Show results
run: |
docker exec promptflow COLUMNS=150 pf run show-details -n base_run
echo "Getting metrics ..."
docker exec promptflow pf run show-metrics -n base_run
##docker exec promptflow pf run visualize -n base_run
echo "Checking results ..."
docker exec promptflow python3 check_evaluation_results.py