From b3d55ff1f18ea291f3af76a62789ab633afe4ec3 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 17 Oct 2024 22:42:28 +0200 Subject: [PATCH] add benchmark --- .github/workflows/ci.yml | 35 ++++++++++++++ CONTRIBUTING.md | 7 +++ benchmarks/README.md | 63 ++++++++++++++++++++++++ benchmarks/benchmarks.py | 96 +++++++++++++++++++++++++++++++++++++ benchmarks/requirements.txt | 6 +++ pyproject.toml | 4 ++ tests/benchmarks.py | 43 +++++++++++++++++ 7 files changed, 254 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/benchmarks.py create mode 100644 benchmarks/requirements.txt create mode 100644 tests/benchmarks.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce69c82..a368421 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,41 @@ jobs: name: Python${{ matrix.python-version }} fail_ci_if_error: false + benchmark: + needs: [tests] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[benchmark]" + + - name: Run Benchmark + run: | + python -m pytest tests/benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min' --benchmark-json output.json + + - name: Store and Compare benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: morecantile Benchmarks + tool: 'pytest' + output-file-path: output.json + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: false + # GitHub API token to make a commit comment + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-pages-branch: 'gh-benchmarks' + # Make a commit on `gh-pages` only if main + auto-push: ${{ github.ref == 'refs/heads/main' }} + benchmark-data-dir-path: dev/benchmarks + publish: needs: [tests] runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7ea8ce0..d443410 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,13 @@ This repo is set to use `pre-commit` to run *isort*, *flake8*, *pydocstring*, *b $ pre-commit install ``` +##### Performance tests + +```sh +python -m pip install -e ".[benchmark]" +python -m pytest tests/benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min' +``` + ### Docs ```bash diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..a2d4f66 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,63 @@ + +## Benchmark + +Compare `mercantile`, `utiles` and `morecantile` + +```sh +python -m pip install -r requirements.txt +python -m pytest benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min' +``` + +``` +----------------------------------------------------- benchmark 'bounds': 21 tests ---------------------------------------------------- +Name (time in ns) Min Max Mean Median +--------------------------------------------------------------------------------------------------------------------------------------- +test_bounds[utiles-(486, 332, 30)] 83.0000 (1.0) 52,000.0000 (52.97) 195.3371 (1.48) 208.0000 (1.60) +test_bounds[utiles-(0, 0, 0)] 83.0000 (1.00) 31,167.0000 (31.75) 193.5957 (1.47) 208.0000 (1.60) +test_bounds[utiles-(1, 0, 1)] 125.0000 (1.51) 981.6700 (1.0) 132.0177 (1.0) 130.8300 (1.01) +test_bounds[utiles-(1, 40, 7)] 125.0000 (1.51) 8,606.2500 (8.77) 132.5298 (1.00) 130.0000 (1.0) +test_bounds[utiles-(486, 332, 10)] 128.3300 (1.55) 1,600.8300 (1.63) 133.4046 (1.01) 131.6700 (1.01) +test_bounds[utiles-(1, 1, 1)] 129.5900 (1.56) 16,846.2500 (17.16) 141.9846 (1.08) 134.1700 (1.03) +test_bounds[utiles-(486, 332, 20)] 138.3300 (1.67) 1,123.3300 (1.14) 142.5420 (1.08) 141.2500 (1.09) +test_bounds[mercantile-(1, 40, 7)] 1,000.0000 (12.05) 92,416.0000 (94.14) 1,193.0112 (9.04) 1,167.0000 (8.98) +test_bounds[mercantile-(1, 1, 1)] 1,075.0000 (12.95) 48,066.8000 (48.96) 1,154.8100 (8.75) 1,141.6000 (8.78) +test_bounds[mercantile-(0, 0, 0)] 1,083.0000 (13.05) 21,041.0000 (21.43) 1,214.6049 (9.20) 1,208.0000 (9.29) +test_bounds[mercantile-(1, 0, 1)] 1,083.0000 (13.05) 75,291.0000 (76.70) 1,213.7837 (9.19) 1,208.0000 (9.29) +test_bounds[mercantile-(486, 332, 10)] 1,083.0000 (13.05) 55,750.0000 (56.79) 1,223.7598 (9.27) 1,208.0000 (9.29) +test_bounds[mercantile-(486, 332, 20)] 1,083.0000 (13.05) 89,041.0000 (90.70) 1,248.4256 (9.46) 1,209.0000 (9.30) +test_bounds[mercantile-(486, 332, 30)] 1,083.0000 (13.05) 194,958.0000 (198.60) 1,239.4339 (9.39) 1,209.0000 (9.30) +test_bounds[morecantile-(1, 0, 1)] 20,083.0000 (241.96) 132,625.0000 (135.10) 20,893.1235 (158.26) 20,667.0000 (158.98) +test_bounds[morecantile-(0, 0, 0)] 20,166.0000 (242.96) 120,333.0000 (122.58) 20,974.1961 (158.87) 20,708.0000 (159.29) +test_bounds[morecantile-(1, 1, 1)] 20,334.0000 (244.99) 60,958.0000 (62.10) 21,236.2953 (160.86) 20,750.0000 (159.62) +test_bounds[morecantile-(1, 40, 7)] 21,667.0000 (261.05) 173,458.0000 (176.70) 22,622.6537 (171.36) 22,333.0000 (171.79) +test_bounds[morecantile-(486, 332, 10)] 22,166.0000 (267.06) 205,125.0000 (208.96) 23,512.1607 (178.10) 22,875.0000 (175.96) +test_bounds[morecantile-(486, 332, 20)] 24,708.0000 (297.69) 129,250.0000 (131.66) 25,775.0796 (195.24) 25,375.0000 (195.19) +test_bounds[morecantile-(486, 332, 30)] 71,292.0000 (858.94) 205,625.0000 (209.46) 73,569.4565 (557.27) 72,667.0000 (558.98) +--------------------------------------------------------------------------------------------------------------------------------------- + +---------------------------------------------------- benchmark 'xy_bounds': 21 tests ----------------------------------------------------- +Name (time in ns) Min Max Mean Median +------------------------------------------------------------------------------------------------------------------------------------------ +test_xy_bounds[utiles-(0, 0, 0)] 83.0000 (1.0) 57,375.0000 (112.55) 145.9127 (1.57) 125.0000 (1.36) +test_xy_bounds[utiles-(1, 1, 1)] 88.3400 (1.06) 1,335.4200 (2.62) 93.1478 (1.00) 92.5000 (1.00) +test_xy_bounds[utiles-(1, 40, 7)] 88.7500 (1.07) 1,451.2500 (2.85) 93.1258 (1.0) 92.0900 (1.0) +test_xy_bounds[utiles-(1, 0, 1)] 89.5900 (1.08) 861.6600 (1.69) 93.6428 (1.01) 93.3300 (1.01) +test_xy_bounds[utiles-(486, 332, 10)] 90.0000 (1.08) 509.7950 (1.0) 93.2818 (1.00) 92.9150 (1.01) +test_xy_bounds[utiles-(486, 332, 20)] 90.4100 (1.09) 864.1700 (1.70) 94.2974 (1.01) 94.1600 (1.02) +test_xy_bounds[utiles-(486, 332, 30)] 90.4100 (1.09) 1,377.5000 (2.70) 94.7366 (1.02) 94.1600 (1.02) +test_xy_bounds[mercantile-(0, 0, 0)] 708.0000 (8.53) 42,250.0000 (82.88) 843.2292 (9.05) 833.0000 (9.05) +test_xy_bounds[mercantile-(486, 332, 10)] 731.2500 (8.81) 5,393.7500 (10.58) 770.8331 (8.28) 764.6000 (8.30) +test_xy_bounds[mercantile-(486, 332, 30)] 731.2500 (8.81) 7,316.6500 (14.35) 770.2423 (8.27) 764.6000 (8.30) +test_xy_bounds[mercantile-(1, 0, 1)] 732.1429 (8.82) 9,404.8571 (18.45) 791.7721 (8.50) 785.7143 (8.53) +test_xy_bounds[mercantile-(1, 40, 7)] 733.3000 (8.83) 5,950.0000 (11.67) 772.3273 (8.29) 766.7000 (8.33) +test_xy_bounds[mercantile-(486, 332, 20)] 733.3500 (8.84) 7,408.3000 (14.53) 773.4718 (8.31) 768.7500 (8.35) +test_xy_bounds[mercantile-(1, 1, 1)] 735.4000 (8.86) 4,762.5000 (9.34) 783.2100 (8.41) 777.1000 (8.44) +test_xy_bounds[morecantile-(0, 0, 0)] 3,042.0000 (36.65) 108,208.0000 (212.26) 3,287.1089 (35.30) 3,250.0000 (35.29) +test_xy_bounds[morecantile-(1, 0, 1)] 3,291.0000 (39.65) 68,166.0000 (133.71) 3,503.4695 (37.62) 3,459.0000 (37.56) +test_xy_bounds[morecantile-(1, 1, 1)] 3,291.0000 (39.65) 72,875.0000 (142.95) 3,506.8544 (37.66) 3,500.0000 (38.01) +test_xy_bounds[morecantile-(1, 40, 7)] 4,375.0000 (52.71) 76,375.0000 (149.82) 4,630.2893 (49.72) 4,584.0000 (49.78) +test_xy_bounds[morecantile-(486, 332, 10)] 4,875.0000 (58.73) 85,208.0000 (167.14) 5,197.3581 (55.81) 5,125.0000 (55.65) +test_xy_bounds[morecantile-(486, 332, 20)] 6,916.0000 (83.33) 78,458.0000 (153.90) 7,221.0669 (77.54) 7,166.0000 (77.82) +test_xy_bounds[morecantile-(486, 332, 30)] 52,667.0000 (634.54) 404,208.0000 (792.88) 54,749.1232 (587.90) 54,333.0000 (590.00) +------------------------------------------------------------------------------------------------------------------------------------------ +``` diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py new file mode 100644 index 0000000..f801a70 --- /dev/null +++ b/benchmarks/benchmarks.py @@ -0,0 +1,96 @@ +"""Morecantile/Mercantile/Utiles comparison benchmark + +The benchmark suite is adapted from jessekrubin/utiles +https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py#L17-L25 +""" +# This file is a modified version of https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py. +# +# The original license follows. +# +# MIT License +# +# Copyright (c) 2023 jessekrubin +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import Callable, Tuple + +import mercantile +import pytest +import utiles + +import morecantile + +tms = morecantile.tms.get("WebMercatorQuad") + +TEST_TILES = ( + (0, 0, 0), + (1, 0, 1), + (1, 1, 1), + (1, 40, 7), + (486, 332, 10), + # HIGH ZOOM + (486, 332, 20), + # OUTSIDE TMS Range + (486, 332, 30), +) + + +@pytest.mark.parametrize( + "tile", + [pytest.param(t, id=str(t)) for t in TEST_TILES], +) +@pytest.mark.parametrize( + "func", + [ + pytest.param(mercantile.bounds, id="mercantile"), + pytest.param(tms.bounds, id="morecantile"), + pytest.param(utiles.bounds, id="utiles"), + ], +) +@pytest.mark.benchmark(group="bounds") +def test_bounds( + tile: Tuple[int, int, int], + func: Callable[[Tuple[int, int, int]], Tuple[float, float]], + benchmark, +) -> None: + """Benchmark bounds() method.""" + _ = benchmark(func, *tile) + + +@pytest.mark.parametrize( + "tile", + [pytest.param(t, id=str(t)) for t in TEST_TILES], +) +@pytest.mark.parametrize( + "func", + [ + pytest.param(mercantile.xy_bounds, id="mercantile"), + pytest.param(tms.xy_bounds, id="morecantile"), + pytest.param(utiles.xy_bounds, id="utiles"), + ], +) +@pytest.mark.benchmark(group="xy_bounds") +def test_xy_bounds( + tile: Tuple[int, int, int], + func: Callable[[Tuple[int, int, int]], Tuple[float, float]], + benchmark, +) -> None: + """Benchmark xy_bounds() method.""" + _ = benchmark(func, *tile) diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt new file mode 100644 index 0000000..e11e111 --- /dev/null +++ b/benchmarks/requirements.txt @@ -0,0 +1,6 @@ +pytest +pytest-benchmark + +morecantile +mercantile +utiles diff --git a/pyproject.toml b/pyproject.toml index d004ae9..fa99611 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,10 @@ test = [ "pytest-cov", "rasterio>=1.2.1", ] +benchmark = [ + "pytest", + "pytest-benchmark", +] dev = [ "pre-commit", "bump-my-version", diff --git a/tests/benchmarks.py b/tests/benchmarks.py new file mode 100644 index 0000000..166d9de --- /dev/null +++ b/tests/benchmarks.py @@ -0,0 +1,43 @@ +"""Morecantile benchmark.""" + +import pytest + +import morecantile +from morecantile.commons import BoundingBox + +tms = morecantile.tms.get("WebMercatorQuad") + +# Test tiles from https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py +TEST_TILES = ( + (0, 0, 0), + (1, 0, 1), + (1, 1, 1), + (1, 40, 7), + (486, 332, 10), + # HIGH ZOOM + (486, 332, 20), + # OUTSIDE TMS Range + (486, 332, 30), +) + + +@pytest.mark.parametrize("tile", TEST_TILES) +def test_bounds(tile, benchmark): + str_tile = "Tile(x={},y={},z={})".format(*tile) + benchmark.name = f"morecantile.bounds-{str_tile}" + benchmark.fullname = f"morecantile.bounds-{str_tile}" + benchmark.group = "morecantile.bounds" + + r = benchmark(tms.bounds, *tile) + assert isinstance(r, BoundingBox) + + +@pytest.mark.parametrize("tile", TEST_TILES) +def test_xy_bounds(tile, benchmark) -> None: + str_tile = "Tile(x={},y={},z={})".format(*tile) + benchmark.name = f"morecantile.xy_bounds-{str_tile}" + benchmark.fullname = f"morecantile.xy_bounds-{str_tile}" + benchmark.group = "morecantile.xy_bounds" + + r = benchmark(tms.xy_bounds, *tile) + assert isinstance(r, BoundingBox)