Skip to content

fix: version compatability fixes #1

fix: version compatability fixes

fix: version compatability fixes #1

name: "QE: WASM benchmarks"
on:
pull_request:
paths-ignore:
- ".github/**"
- "!.github/workflows/wasm-benchmarks.yml"
- ".buildkite/**"
- "*.md"
- "LICENSE"
- "CODEOWNERS"
- "renovate.json"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
benchmarks:
runs-on: ubuntu-latest
env: # Set environment variables for the whole job
PROFILE: release
steps:
- name: Checkout PR branch
uses: actions/checkout@v4
with:
# using head ref rather than merge branch to get original commit message
ref: ${{ github.event.pull_request.head.sha }}
- uses: ./.github/workflows/include/rust-wasm-setup
- name: "Setup Node.js"
uses: actions/setup-node@v4
- name: "Setup pnpm"
uses: pnpm/[email protected]
with:
version: 8
- name: "Login to Docker Hub"
uses: docker/login-action@v3
continue-on-error: true
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
if: "${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}"
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract Branch Name
run: |
echo "Extracting branch name from: $(git show -s --format=%s)"
branch="$(git show -s --format=%s | grep -o "DRIVER_ADAPTERS_BRANCH=[^ ]*" | cut -f2 -d=)"
echo "branch=$branch"
if [ -n "$branch" ]; then
echo "Using $branch branch of driver adapters"
echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV"
fi
- name: Setup benchmark
run: make setup-pg-bench
- name: Run benchmarks
id: bench
if: false
run: |
make run-bench | tee results.txt
# Extract the values from the benchmark output
regressed_values=$(grep "slower than Web Assembly: Latest" results.txt | cut -f1 -d'x')
improved_values=$(grep "faster than Web Assembly: Latest" results.txt | cut -f1 -d'x')
# Initialize sum variable and count
total_sum=0
total_count=0
# Add the inverted regressed values to the sum
for value in $regressed_values; do
inverted=$(echo "scale=4; 1/$value" | bc)
echo "Regressed value: $inverted"
total_sum=$(echo "$total_sum + $inverted" | bc)
total_count=$((total_count + 1))
done
# Add the improved values to the sum
for value in $improved_values; do
echo "Improved value: $value"
total_sum=$(echo "$total_sum + $value" | bc)
total_count=$((total_count + 1))
done
if [ $total_count -eq 0 ]; then
echo "πŸ’₯ something was wrong running the benchmarks"
exit 1
fi
mean=$(echo "scale=4; $total_sum / $total_count" | bc)
echo "Extracted $total_count values from the benchmark output"
echo "Total sum: $total_sum"
echo "Total count: $total_count"
echo "Mean: $mean"
# Report improvement or worsening. Fails if >= 1.5% worsening.
if (( $(echo "$mean < 0.985" | bc -l) )); then
percent=$(echo "scale=4; ((1 / $mean) - 1) * 100" | bc)
summary="❌ WASM query-engine performance will worsen by $(printf %.2f "$percent")%"
status=failed
elif (( $(echo "$mean > 1.015" | bc -l) )); then
percent=$(echo "scale=4; ($mean - 1) * 100" | bc)
summary="πŸš€ WASM query-engine performance will improve by $(printf %.2f "$percent")%"
status=passed
else
delta=$(echo "scale=3; (1 / $mean)" | bc)
summary="βœ… WASM query-engine performance won't change substantially ($(printf %.3f "$delta")x)"
status=passed
fi
echo "summary=$summary" >> "$GITHUB_OUTPUT"
echo "status=$status" >> "$GITHUB_OUTPUT"
# Save the output to a file so we can use it in the comment
{
echo 'bench_output<<EOF'
cat results.txt
echo EOF
} >> "$GITHUB_OUTPUT"
- name: Find past report comment
if: false
uses: peter-evans/find-comment@v3
id: findReportComment
with:
issue-number: ${{ github.event.pull_request.number }}
body-includes: "<!-- wasm-engine-perf -->"
- name: Create or update report
uses: peter-evans/create-or-update-comment@v4
# Only run on branches from our repository
# It avoids an expected failure on forks
# if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
if: false
with:
comment-id: ${{ steps.findReportComment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
<!-- wasm-engine-perf -->
#### ${{ steps.bench.outputs.summary }}
<details>
<summary>Full benchmark report</summary>
```
${{ steps.bench.outputs.bench_output }}
```
</details>
After changes in ${{ github.event.pull_request.head.sha }}
edit-mode: replace
- name: Fail workflow if regression detected
# if: steps.bench.outputs.status == 'failed'
if: false
run: |
echo "Workflow failed due to benchmark regression."
exit 1