From fcdae87a60b46624e4be71a50a4fda02e6b78892 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 25 Sep 2020 01:45:59 +0200 Subject: [PATCH] Fix official Docker image building by using wheel from working tree This will invoke "poetry build --format=wheel" and will install that wheel package inside the Docker container instead of acquiring the package from PyPI. It also adjusts the selection of dependencies for wetterdienst-standard and wetterdienst-full images. --- .dockerignore | 2 + .github/release/full.test.yml | 3 +- .github/release/full/Dockerfile | 12 ++++- .github/release/standard.test.yml | 3 +- .github/release/standard/Dockerfile | 6 ++- .github/workflows/docker-publish-full.yml | 31 +++++++++-- .github/workflows/docker-publish-standard.yml | 31 +++++++++-- poetry.lock | 53 +++++++++++++------ pyproject.toml | 4 +- 9 files changed, 113 insertions(+), 32 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2aafc4b7b..9ff4bc0b7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,5 @@ !pytest.ini !noxfile.py !.coverragerc + +!dist diff --git a/.github/release/full.test.yml b/.github/release/full.test.yml index 34879b440..3effe5d1b 100644 --- a/.github/release/full.test.yml +++ b/.github/release/full.test.yml @@ -1,4 +1,5 @@ sut: - build: full + build: ../.. + dockerfile: .github/release/full/Dockerfile # TODO: We might went to improve this. Just say "wetterdienst about system" here. command: python -c 'import wradlib; print(wradlib.__version__)' diff --git a/.github/release/full/Dockerfile b/.github/release/full/Dockerfile index af48946aa..b99d93d44 100644 --- a/.github/release/full/Dockerfile +++ b/.github/release/full/Dockerfile @@ -15,7 +15,7 @@ ENV TERM linux # Install GDAL. RUN apt-get update -RUN apt-get --yes install build-essential libgdal-dev +RUN apt-get --yes install build-essential libgdal-dev libmariadbclient-dev # Make sure you have numpy installed before you attempt to install the GDAL Python bindings. # https://gis.stackexchange.com/a/274328 @@ -25,6 +25,9 @@ RUN pip install GDAL==$(gdal-config --version) # Install wradlib. RUN pip install wradlib +# Install database adapters +RUN pip install mysqlclient + # 2. Main FROM python:3.8.5-slim @@ -37,4 +40,9 @@ RUN apt-get --yes install libgdal20 COPY --from=build-step /usr/local/lib /usr/local/lib # Install Wetterdienst. -RUN pip install wetterdienst[excel] + +# Use "poetry build --format=wheel" to build wheel packages. +COPY dist/*.whl /tmp/ + +# Install latest wheel package. +RUN pip install $(ls -c /tmp/wetterdienst-*-py3-none-any.whl)[http,sql,excel,duckdb,influxdb,cratedb,mysql,postgresql] diff --git a/.github/release/standard.test.yml b/.github/release/standard.test.yml index e382fad41..7562c6c38 100644 --- a/.github/release/standard.test.yml +++ b/.github/release/standard.test.yml @@ -1,3 +1,4 @@ sut: - build: standard + build: ../.. + dockerfile: .github/release/standard/Dockerfile command: wetterdienst about parameters diff --git a/.github/release/standard/Dockerfile b/.github/release/standard/Dockerfile index 0df25b0f3..90d899112 100644 --- a/.github/release/standard/Dockerfile +++ b/.github/release/standard/Dockerfile @@ -3,4 +3,8 @@ FROM python:3.8.5-slim ENV DEBIAN_FRONTEND noninteractive ENV TERM linux -RUN pip install wetterdienst[excel] +# Use "poetry build --format=wheel" to build wheel packages. +COPY dist/*.whl /tmp/ + +# Install latest wheel package. +RUN pip install $(ls -c /tmp/wetterdienst-*-py3-none-any.whl)[http,sql,excel] diff --git a/.github/workflows/docker-publish-full.yml b/.github/workflows/docker-publish-full.yml index 954a53928..e53581c3a 100644 --- a/.github/workflows/docker-publish-full.yml +++ b/.github/workflows/docker-publish-full.yml @@ -5,7 +5,6 @@ on: # Publish `master` as Docker `latest` image. branches: - master - - test-docker # Publish `v1.2.3` tags as releases. tags: @@ -24,7 +23,19 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Acquire sources + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install Poetry + run: pip install poetry==1.0.10 + + - name: Build wheel package + run: poetry build --format=wheel - name: Run tests run: | @@ -43,9 +54,21 @@ jobs: if: github.event_name == 'push' steps: - - uses: actions/checkout@v2 + - name: Acquire sources + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install Poetry + run: pip install poetry==1.0.10 + + - name: Build wheel package + run: poetry build --format=wheel - - name: Build image + - name: Build Docker image run: docker build . --file .github/release/full/Dockerfile --tag $IMAGE_NAME - name: Log into GitHub Container Registry diff --git a/.github/workflows/docker-publish-standard.yml b/.github/workflows/docker-publish-standard.yml index 3d2d71eae..19b7c657f 100644 --- a/.github/workflows/docker-publish-standard.yml +++ b/.github/workflows/docker-publish-standard.yml @@ -5,7 +5,6 @@ on: # Publish `master` as Docker `latest` image. branches: - master - - test-docker # Publish `v1.2.3` tags as releases. tags: @@ -24,7 +23,19 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Acquire sources + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install Poetry + run: pip install poetry==1.0.10 + + - name: Build wheel package + run: poetry build --format=wheel - name: Run tests run: | @@ -43,9 +54,21 @@ jobs: if: github.event_name == 'push' steps: - - uses: actions/checkout@v2 + - name: Acquire sources + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install Poetry + run: pip install poetry==1.0.10 + + - name: Build wheel package + run: poetry build --format=wheel - - name: Build image + - name: Build Docker image run: docker build . --file .github/release/standard/Dockerfile --tag $IMAGE_NAME - name: Log into GitHub Container Registry diff --git a/poetry.lock b/poetry.lock index e4d9863b4..897d85bf8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1193,7 +1193,7 @@ wcwidth = "*" [[package]] category = "main" description = "psycopg2 - Python-PostgreSQL Database Adapter" -name = "psycopg2" +name = "psycopg2-binary" optional = true python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" version = "2.8.6" @@ -1962,11 +1962,11 @@ http = ["fastapi", "uvicorn"] influxdb = ["influxdb"] ipython = ["ipython", "ipython-genutils", "matplotlib"] mysql = ["mysqlclient"] -postgresql = ["psycopg2"] +postgresql = ["psycopg2-binary"] sql = ["duckdb"] [metadata] -content-hash = "0d2f1efe3dfd162f73221de91218371872ce5fe5b35db0ba3791e3d9e0d4d3e4" +content-hash = "eeb3b87cfd01cc2557a1e7dc108738beecf726aaeab24b49a205874308d8c270" lock-version = "1.0" python-versions = "^3.6.1" @@ -2680,20 +2680,39 @@ prompt-toolkit = [ {file = "prompt_toolkit-3.0.7-py3-none-any.whl", hash = "sha256:83074ee28ad4ba6af190593d4d4c607ff525272a504eb159199b6dd9f950c950"}, {file = "prompt_toolkit-3.0.7.tar.gz", hash = "sha256:822f4605f28f7d2ba6b0b09a31e25e140871e96364d1d377667b547bb3bf4489"}, ] -psycopg2 = [ - {file = "psycopg2-2.8.6-cp27-cp27m-win32.whl", hash = "sha256:068115e13c70dc5982dfc00c5d70437fe37c014c808acce119b5448361c03725"}, - {file = "psycopg2-2.8.6-cp27-cp27m-win_amd64.whl", hash = "sha256:d160744652e81c80627a909a0e808f3c6653a40af435744de037e3172cf277f5"}, - {file = "psycopg2-2.8.6-cp34-cp34m-win32.whl", hash = "sha256:b8cae8b2f022efa1f011cc753adb9cbadfa5a184431d09b273fb49b4167561ad"}, - {file = "psycopg2-2.8.6-cp34-cp34m-win_amd64.whl", hash = "sha256:f22ea9b67aea4f4a1718300908a2fb62b3e4276cf00bd829a97ab5894af42ea3"}, - {file = "psycopg2-2.8.6-cp35-cp35m-win32.whl", hash = "sha256:26e7fd115a6db75267b325de0fba089b911a4a12ebd3d0b5e7acb7028bc46821"}, - {file = "psycopg2-2.8.6-cp35-cp35m-win_amd64.whl", hash = "sha256:00195b5f6832dbf2876b8bf77f12bdce648224c89c880719c745b90515233301"}, - {file = "psycopg2-2.8.6-cp36-cp36m-win32.whl", hash = "sha256:a49833abfdede8985ba3f3ec641f771cca215479f41523e99dace96d5b8cce2a"}, - {file = "psycopg2-2.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:f974c96fca34ae9e4f49839ba6b78addf0346777b46c4da27a7bf54f48d3057d"}, - {file = "psycopg2-2.8.6-cp37-cp37m-win32.whl", hash = "sha256:6a3d9efb6f36f1fe6aa8dbb5af55e067db802502c55a9defa47c5a1dad41df84"}, - {file = "psycopg2-2.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:56fee7f818d032f802b8eed81ef0c1232b8b42390df189cab9cfa87573fe52c5"}, - {file = "psycopg2-2.8.6-cp38-cp38-win32.whl", hash = "sha256:ad2fe8a37be669082e61fb001c185ffb58867fdbb3e7a6b0b0d2ffe232353a3e"}, - {file = "psycopg2-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:56007a226b8e95aa980ada7abdea6b40b75ce62a433bd27cec7a8178d57f4051"}, - {file = "psycopg2-2.8.6.tar.gz", hash = "sha256:fb23f6c71107c37fd667cb4ea363ddeb936b348bbd6449278eb92c189699f543"}, +psycopg2-binary = [ + {file = "psycopg2-binary-2.8.6.tar.gz", hash = "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0"}, + {file = "psycopg2_binary-2.8.6-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4"}, + {file = "psycopg2_binary-2.8.6-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db"}, + {file = "psycopg2_binary-2.8.6-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5"}, + {file = "psycopg2_binary-2.8.6-cp27-cp27m-win32.whl", hash = "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25"}, + {file = "psycopg2_binary-2.8.6-cp27-cp27m-win_amd64.whl", hash = "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c"}, + {file = "psycopg2_binary-2.8.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c"}, + {file = "psycopg2_binary-2.8.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1"}, + {file = "psycopg2_binary-2.8.6-cp34-cp34m-win32.whl", hash = "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2"}, + {file = "psycopg2_binary-2.8.6-cp34-cp34m-win_amd64.whl", hash = "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152"}, + {file = "psycopg2_binary-2.8.6-cp35-cp35m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449"}, + {file = "psycopg2_binary-2.8.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859"}, + {file = "psycopg2_binary-2.8.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550"}, + {file = "psycopg2_binary-2.8.6-cp35-cp35m-win32.whl", hash = "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd"}, + {file = "psycopg2_binary-2.8.6-cp35-cp35m-win_amd64.whl", hash = "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71"}, + {file = "psycopg2_binary-2.8.6-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4"}, + {file = "psycopg2_binary-2.8.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb"}, + {file = "psycopg2_binary-2.8.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da"}, + {file = "psycopg2_binary-2.8.6-cp36-cp36m-win32.whl", hash = "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2"}, + {file = "psycopg2_binary-2.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a"}, + {file = "psycopg2_binary-2.8.6-cp37-cp37m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679"}, + {file = "psycopg2_binary-2.8.6-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf"}, + {file = "psycopg2_binary-2.8.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b"}, + {file = "psycopg2_binary-2.8.6-cp37-cp37m-win32.whl", hash = "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67"}, + {file = "psycopg2_binary-2.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66"}, + {file = "psycopg2_binary-2.8.6-cp38-cp38-macosx_10_9_x86_64.macosx_10_9_intel.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f"}, + {file = "psycopg2_binary-2.8.6-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77"}, + {file = "psycopg2_binary-2.8.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94"}, + {file = "psycopg2_binary-2.8.6-cp38-cp38-win32.whl", hash = "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729"}, + {file = "psycopg2_binary-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd"}, ] ptyprocess = [ {file = "ptyprocess-0.6.0-py2.py3-none-any.whl", hash = "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"}, diff --git a/pyproject.toml b/pyproject.toml index ad32a2ab4..94672c0d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ duckdb = { version = "^0.2.1", optional = true } influxdb = { version = "^5.3.0", optional = true } crate = { version = "^0.25.0", optional = true, extras = ["sqlalchemy"] } mysqlclient = { version = "^2.0.1", optional = true } -psycopg2 = { version = "^2.8.6", optional = true } +psycopg2-binary = { version = "^2.8.6", optional = true } fastapi = { version = "^0.61.1", optional = true } uvicorn = { version = "^0.11.8", optional = true } @@ -125,7 +125,7 @@ duckdb = ["duckdb"] influxdb = ["influxdb"] cratedb = ["crate"] mysql = ["mysqlclient"] -postgresql = ["psycopg2"] +postgresql = ["psycopg2-binary"] [tool.poetry.dev-dependencies] nox = "^2020.8.22"