ci improvements (#162)

.github/workflows/pythonapp.yml

@@ -32,7 +32,6 @@ jobs:
           pip install -r requirements.txt
           pip install -r requirements-test.txt
           pip install pytest
-          pip install tox tox-gh-actions
       #     - name: Lint with flake8
       #       run: |
       #         pip install flake8
@@ -45,6 +44,6 @@ jobs:
           pycodestyle -v awscurl
       - name: Test with pytest
         run: |
+          python --version
           pytest -v --cov=awscurl --cov-fail-under=77 --cov-report html
-      - name: Test with tox
-        run: tox -vv
+

ci/ci-alpine/Dockerfile

@@ -20,17 +20,19 @@ ENV PATH="/root/.pyenv/bin:$PATH"
 RUN eval "$(pyenv init -)" && \
     eval "$(pyenv virtualenv-init -)"
 
-COPY .python-version .python-version
-RUN cat .python-version | xargs -n 1 /root/.pyenv/bin/pyenv install
-
-RUN /root/.pyenv/versions/3.8*/bin/python -m venv /root/venv
-
-RUN bash -c "source /root/venv/bin/activate && pip install tox"
 
-COPY awscurl dd/awscurl
-COPY tox.ini dd/tox.ini
-COPY requirements.txt dd/requirements.txt
-COPY requirements-test.txt dd/requirements-test.txt
-COPY tests dd/tests
-
-RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"
+WORKDIR /root/workdir
+COPY .python-version .python-version
+RUN for version in $(cat .python-version); do \
+    /root/.pyenv/bin/pyenv install $version; \
+done
+
+COPY setup.cfg setup.cfg
+COPY awscurl awscurl
+COPY setup.py setup.py
+COPY scripts/ci.sh scripts/ci.sh
+COPY requirements.txt requirements.txt
+COPY requirements-test.txt requirements-test.txt
+COPY tests tests
+
+# RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"

ci/ci-centos/Dockerfile

@@ -8,21 +8,22 @@ RUN yum install -y libffi-devel readline-devel zlib-devel bzip2-devel sqlite-dev
 RUN curl https://pyenv.run | bash
 
 ENV PATH="/root/.pyenv/bin:$PATH"
-
 RUN eval "$(pyenv init -)" && \
     eval "$(pyenv virtualenv-init -)"
 
-COPY .python-version .python-version
-RUN cat .python-version | xargs -n 1 /root/.pyenv/bin/pyenv install
-
-RUN /root/.pyenv/versions/3.8*/bin/python -m venv /root/venv
 
-RUN bash -c "source /root/venv/bin/activate && pip install tox"
-
-COPY awscurl dd/awscurl
-COPY tox.ini dd/tox.ini
-COPY requirements.txt dd/requirements.txt
-COPY requirements-test.txt dd/requirements-test.txt
-COPY tests dd/tests
-
-RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"
+WORKDIR /root/workdir
+COPY .python-version .python-version
+RUN for version in $(cat .python-version); do \
+    /root/.pyenv/bin/pyenv install $version; \
+done
+
+COPY setup.cfg setup.cfg
+COPY awscurl awscurl
+COPY setup.py setup.py
+COPY scripts/ci.sh scripts/ci.sh
+COPY requirements.txt requirements.txt
+COPY requirements-test.txt requirements-test.txt
+COPY tests tests
+
+# RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"

ci/ci-ubuntu/Dockerfile

@@ -18,21 +18,22 @@ RUN apt install -y curl git \
 RUN curl https://pyenv.run | bash
 
 ENV PATH="/root/.pyenv/bin:$PATH"
-
 RUN eval "$(pyenv init -)" && \
     eval "$(pyenv virtualenv-init -)"
 
-COPY .python-version .python-version
-RUN cat .python-version | xargs -n 1 /root/.pyenv/bin/pyenv install
-
-RUN /root/.pyenv/versions/3.8*/bin/python -m venv /root/venv
 
-RUN bash -c "source /root/venv/bin/activate && pip install tox"
-
-COPY awscurl dd/awscurl
-COPY tox.ini dd/tox.ini
-COPY requirements.txt dd/requirements.txt
-COPY requirements-test.txt dd/requirements-test.txt
-COPY tests dd/tests
-
-RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"
+WORKDIR /root/workdir
+COPY .python-version .python-version
+RUN for version in $(cat .python-version); do \
+    /root/.pyenv/bin/pyenv install $version; \
+done
+
+COPY setup.cfg setup.cfg
+COPY awscurl awscurl
+COPY setup.py setup.py
+COPY scripts/ci.sh scripts/ci.sh
+COPY requirements.txt requirements.txt
+COPY requirements-test.txt requirements-test.txt
+COPY tests tests
+
+# RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"

requirements-test.txt

@@ -1,4 +1,8 @@
 pycodestyle
 mock
 pytest
-pytest-cov
+pytest-cov
+wheel
+setuptools
+setuptools-rust
+build

scripts/ci-in-docker.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -o xtrace
+
+docker_run() {
+    local image_type=$1
+    local script_file=$2
+    echo building "${image_type}" image -- first time it could take a few minutes
+    docker build -t awscurl-ci-"$image_type" -f ./ci/ci-"$image_type"/Dockerfile . && \
+    docker run -t awscurl-ci-"$image_type" bash -c "$script_file"
+}
+
+docker_run "ubuntu" "./scripts/ci.sh"
+docker_run "alpine" "./scripts/ci.sh"
+docker_run "centos" "./scripts/ci.sh"

scripts/ci.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+DETOX_ROOT_DIR=./build/detox
+
+grep -v '^ *#' < .python-version | while IFS= read -r PYENV_VERSION
+do
+    # echo PYENV_VERSION="$PYENV_VERSION"
+    # echo SHELL="$SHELL"
+    # echo $PATH
+    # pyenv install -sv "${PYENV_VERSION}"
+    # https://github.com/pyenv/pyenv/issues/1819#issuecomment-780803524
+    # /root/.pyenv/bin/pyenv shell "${PYENV_VERSION}"
+    export PYENV_VERSION="$PYENV_VERSION" 
+    eval "$(pyenv init -)"
+
+    PER_VER_DIR=${DETOX_ROOT_DIR}/v${PYENV_VERSION}
+    VENV_DIR=${PER_VER_DIR}/venv${PYENV_VERSION}
+    (
+        echo "##### NEW DETOX ENV: " "$(uname) " "${PER_VER_DIR}" " #####"
+        python3 -m venv "${VENV_DIR}"
+        source "${VENV_DIR}"/bin/activate
+
+        echo which python="$(which python)"
+        echo python --version="$(python --version)"
+        echo pip --version="$(pip --version)"
+
+        PS4='[$(date "+%Y-%m-%d %H:%M:%S")] '
+        set -o errexit -o pipefail -o nounset -o xtrace
+
+        pip -q -q install --upgrade pip
+        # python -m ensurepip --upgrade
+        # pip install -r requirements.txt
+        pip -q -q install -r requirements-test.txt
+
+        pycodestyle .
+
+        # python -m build . 
+        pip -q install . 
+
+        export AWS_ACCESS_KEY_ID=MOCK_AWS_ACCESS_KEY_ID
+        export AWS_SECRET_ACCESS_KEY=MOCK_AWS_SECRET_ACCESS_KEY
+        export AWS_SESSION_TOKEN=MOCK_AWS_SESSION_TOKEN
+
+        pytest \
+            --cov=awscurl \
+            --cov-fail-under=77 \
+            --cov-report html \
+            --cov-report=html:"${PER_VER_DIR}"/htmlcov \
+            --durations=2 \
+            --strict-config
+    )
+done

install.sh → scripts/install.sh

@@ -41,4 +41,3 @@ grep -v '^ *#' < .python-version | while IFS= read -r line
 do
   pyenv install -s "${line}"
 done
-pip install tox

setup.cfg

@@ -2,3 +2,5 @@
 count = False
 max-line-length = 120
 statistics = True
+ignore = E501,W291,E225,E123,E266,W504
+exclude = build,venv,venv3.6.15,venv3.8.16,.tox

setup.py

@@ -2,6 +2,12 @@
 
 from setuptools import setup
 
+
+with open("requirements.txt", "r", encoding="utf-8") as f:
+    requirements = f.read().splitlines()
+
+# https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html#summary
+
 setup(
     name='awscurl',
     version='0.27',
@@ -20,12 +26,7 @@
         ],
     },
     zip_safe=False,
-    install_requires=[
-        'requests',
-        'configargparse',
-        'configparser',
-        'urllib3[secure]'
-    ],
+    install_requires=requirements,
     extras_require={
         'awslibs': ["botocore"]
     }