Add PGO wheels build

emmett-framework · Nov 10, 2023 · 36c1ee6 · 36c1ee6
1 parent 4d47abc
commit 36c1ee6
Show file tree

Hide file tree

Showing 6 changed files with 192 additions and 4 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -19,8 +19,17 @@ jobs:
         include:
           - os: ubuntu
             platform: linux
+          - os: ubuntu
+            platform: linux
+            target: x86_64
+            manylinux: auto
+            interpreter: pypy3.8 pypy3.9 pypy3.10
+          - os: macos
+            target: x86_64
+            interpreter: pypy3.8 pypy3.9 pypy3.10
           - os: macos
-            interpreter: 3.8 3.9 3.10 3.11 3.12 pypy3.9 pypy3.10
+            target: aarch64
+            interpreter: 3.8 3.9 pypy3.8 pypy3.9 pypy3.10
           - os: ubuntu
             platform: linux
             target: aarch64
@@ -33,6 +42,9 @@ jobs:
             platform: linux
             target: aarch64
             manylinux: musllinux_1_1
+          - os: windows
+            target: x86_64
+            interpreter: pypy3.8 pypy3.9 pypy3.10
         exclude:
           - os: windows
             target: aarch64
@@ -52,3 +64,65 @@ jobs:
         with:
           name: dist
           path: dist
+
+  wheels-pgo:
+    name: pgo-wheel ${{ matrix.os }} (${{ matrix.interpreter}})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, macos-latest-xlarge, windows-latest]
+        manylinux: [auto]
+        interpreter: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        include:
+          - os: ubuntu-latest
+            platform: linux
+        exclude:
+          - os: macos-latest-xlarge
+            interpreter: '3.8'
+          - os: macos-latest-xlarge
+            interpreter: '3.9'
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.interpreter }}
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: llvm-tools
+      - run: |
+          RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/$RUST_HOST/bin/llvm-profdata >> "$GITHUB_ENV"'
+        shell: bash
+      - name: Build initial wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: v${{ env.MATURIN_VERSION }}
+          rust-toolchain: stable
+          command: build
+          args: --release --out pgo_wheel --interpreter ${{ matrix.interpreter }}
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+        env:
+          RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"
+      - name: Generate PGO data
+        run: |
+          pip install pytest pytest-asyncio httpx websockets
+          pip install granian --no-index --no-deps --find-links pgo_wheel --force-reinstall
+          PGO_RUN=y pytest tests
+      - name: merge PGO data
+        run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
+      - name: Build PGO wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: v${{ env.MATURIN_VERSION }}
+          command: build
+          args: --release --out dist --interpreter ${{ matrix.interpreter }}
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+          rust-toolchain: stable
+        env:
+          RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -42,8 +42,17 @@ jobs:
         include:
           - os: ubuntu
             platform: linux
+          - os: ubuntu
+            platform: linux
+            target: x86_64
+            manylinux: auto
+            interpreter: pypy3.8 pypy3.9 pypy3.10
+          - os: macos
+            target: x86_64
+            interpreter: pypy3.8 pypy3.9 pypy3.10
           - os: macos
-            interpreter: 3.8 3.9 3.10 3.11 3.12 pypy3.9 pypy3.10
+            target: aarch64
+            interpreter: 3.8 3.9 pypy3.8 pypy3.9 pypy3.10
           - os: ubuntu
             platform: linux
             target: aarch64
@@ -56,6 +65,9 @@ jobs:
             platform: linux
             target: aarch64
             manylinux: musllinux_1_1
+          - os: windows
+            target: x86_64
+            interpreter: pypy3.8 pypy3.9 pypy3.10
         exclude:
           - os: windows
             target: aarch64
@@ -76,9 +88,71 @@ jobs:
           name: dist
           path: dist
 
+  wheels-pgo:
+    name: pgo-wheel ${{ matrix.os }} (${{ matrix.interpreter}})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, macos-latest-xlarge, windows-latest]
+        manylinux: [auto]
+        interpreter: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        include:
+          - os: ubuntu-latest
+            platform: linux
+        exclude:
+          - os: macos-latest-xlarge
+            interpreter: '3.8'
+          - os: macos-latest-xlarge
+            interpreter: '3.9'
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.interpreter }}
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: llvm-tools
+      - run: |
+          RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/$RUST_HOST/bin/llvm-profdata >> "$GITHUB_ENV"'
+        shell: bash
+      - name: Build initial wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: v${{ env.MATURIN_VERSION }}
+          rust-toolchain: stable
+          command: build
+          args: --release --out pgo_wheel --interpreter ${{ matrix.interpreter }}
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+        env:
+          RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"
+      - name: Generate PGO data
+        run: |
+          pip install pytest pytest-asyncio httpx websockets
+          pip install granian --no-index --no-deps --find-links pgo_wheel --force-reinstall
+          PGO_RUN=y pytest tests
+      - name: merge PGO data
+        run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
+      - name: Build PGO wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: v${{ env.MATURIN_VERSION }}
+          command: build
+          args: --release --out dist --interpreter ${{ matrix.interpreter }}
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+          rust-toolchain: stable
+        env:
+          RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+
   release:
     runs-on: ubuntu-latest
-    needs: [ sdist, wheels ]
+    needs: [ sdist, wheels, wheels-pgo ]
     steps:
       - uses: actions/download-artifact@v3
         with:

diff --git a/tests/test_asgi.py b/tests/test_asgi.py
@@ -1,8 +1,11 @@
+import os
+
 import httpx
 import pytest
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_scope(asgi_server, threading_mode):
     async with asgi_server(threading_mode) as port:
@@ -44,6 +47,7 @@ async def test_body_large(asgi_server, threading_mode):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_app_error(asgi_server, threading_mode):
     async with asgi_server(threading_mode) as port:
@@ -53,6 +57,7 @@ async def test_app_error(asgi_server, threading_mode):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_protocol_error(asgi_server, threading_mode):
     async with asgi_server(threading_mode) as port:

diff --git a/tests/test_rsgi.py b/tests/test_rsgi.py
@@ -1,8 +1,11 @@
+import os
+
 import httpx
 import pytest
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_scope(rsgi_server, threading_mode):
     async with rsgi_server(threading_mode) as port:
@@ -32,6 +35,18 @@ async def test_body(rsgi_server, threading_mode):
     assert res.text == 'test'
 
 
+@pytest.mark.asyncio
+@pytest.mark.skipif(not bool(os.getenv('PGO_RUN')), reason='not PGO build')
+@pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
+async def test_body_large(rsgi_server, threading_mode):
+    data = ''.join([f'{idx}test'.zfill(8) for idx in range(0, 5000)])
+    async with rsgi_server(threading_mode) as port:
+        res = httpx.post(f'http://localhost:{port}/echo', content=data)
+
+    assert res.status_code == 200
+    assert res.text == data
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_body_stream_req(rsgi_server, threading_mode):
@@ -54,6 +69,7 @@ async def test_body_stream_res(rsgi_server, threading_mode):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_app_error(rsgi_server, threading_mode):
     async with rsgi_server(threading_mode) as port:

diff --git a/tests/test_ws.py b/tests/test_ws.py
@@ -1,12 +1,13 @@
 import json
+import os
 import sys
 
 import pytest
 import websockets
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='skip on windows')
 @pytest.mark.asyncio
+@pytest.mark.skipif(sys.platform == 'win32', reason='skip on windows')
 @pytest.mark.parametrize('server', ['asgi', 'rsgi'], indirect=True)
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_messages(server, threading_mode):
@@ -34,6 +35,7 @@ async def test_reject(server, threading_mode):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_asgi_scope(asgi_server, threading_mode):
     async with asgi_server(threading_mode) as port:
@@ -51,6 +53,7 @@ async def test_asgi_scope(asgi_server, threading_mode):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_rsgi_scope(rsgi_server, threading_mode):
     async with rsgi_server(threading_mode) as port:

diff --git a/tests/test_wsgi.py b/tests/test_wsgi.py
@@ -1,8 +1,11 @@
+import os
+
 import httpx
 import pytest
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_scope(wsgi_server, threading_mode):
     payload = 'body_payload'
@@ -31,6 +34,18 @@ async def test_body(wsgi_server, threading_mode):
     assert res.text == 'test'
 
 
+@pytest.mark.asyncio
+@pytest.mark.skipif(not bool(os.getenv('PGO_RUN')), reason='not PGO build')
+@pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
+async def test_body_large(wsgi_server, threading_mode):
+    data = ''.join([f'{idx}test'.zfill(8) for idx in range(0, 5000)])
+    async with wsgi_server(threading_mode) as port:
+        res = httpx.post(f'http://localhost:{port}/echo', content=data)
+
+    assert res.status_code == 200
+    assert res.text == data
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_iterbody(wsgi_server, threading_mode):
@@ -42,6 +57,7 @@ async def test_iterbody(wsgi_server, threading_mode):
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(bool(os.getenv('PGO_RUN')), reason='PGO build')
 @pytest.mark.parametrize('threading_mode', ['runtime', 'workers'])
 async def test_app_error(wsgi_server, threading_mode):
     async with wsgi_server(threading_mode) as port: