From 5fe1fb10beab91d06846f8118520710ef58f779b Mon Sep 17 00:00:00 2001 From: Florian Scherf Date: Mon, 29 May 2023 11:27:23 +0200 Subject: [PATCH] tests: refactor docker setup for local testing The previous setup for local testing, using docker and tox, had multiple problems: 1) Tests ran as root in the container. All files, created by tox in the container belonged to root after the test-suite finished. That meant that caches and test artifacts, like screenshots or clips, had to be accessed using sudo. 2) Playwright installation steps could not be cached. The playwright installation commands, that setup the underlying ubuntu container and install the actual browsers, had to run on every run of the test-suite, which is expensive. 3) The test container had only one, unspecified, Python version installed. The docker image for the test-suite is based on the Microsofts playwright image, which is based some Ubuntu image. So the Python version always was the current Python3 version on Ubuntu. This was problematic because the version could change on every run and testing with multiple Python versions was impossible. This patch fixes all of these problems by adding a script that runs the test container as the current user, and updates the playwright docker file to install all Python versions that Lona supports, using pyenv. Signed-off-by: Florian Scherf --- .github/workflows/ci.yml | 2 ++ Makefile | 29 +++++++++-------------------- docker-compose | 7 +++++++ docker-compose.yml | 25 +++++++++++++++++++++++++ playwright.Dockerfile | 30 ++++++++++++++++++++++++++++++ tests/docker/docker-compose.yml | 22 ---------------------- tests/docker/playwright.dockerfile | 4 ---- tox.ini | 6 ++---- 8 files changed, 75 insertions(+), 50 deletions(-) create mode 100755 docker-compose create mode 100644 docker-compose.yml create mode 100644 playwright.Dockerfile delete mode 100644 tests/docker/docker-compose.yml delete mode 100644 tests/docker/playwright.dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db38aa41..c5e4630a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,8 @@ jobs: run: | pip install -U setuptools pip install -r tests/REQUIREMENTS.test.txt + python -m playwright install + python -m playwright install-deps - name: Run Tox # Run tox using the version of Python in `PATH` diff --git a/Makefile b/Makefile index 3ee8403c..09304207 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ PYTHON_ENV_ROOT=envs PYTHON_DEV_ENV=$(PYTHON_ENV_ROOT)/$(PYTHON)-dev PYTHON_PACKAGING_ENV=$(PYTHON_ENV_ROOT)/$(PYTHON)-packaging-env -.PHONY: clean doc dist pytest test ci-test lint isort shell freeze +.PHONY: clean doc dist test ci-test lint isort shell freeze # development environment ##################################################### $(PYTHON_DEV_ENV): REQUIREMENTS.dev.txt @@ -36,28 +36,17 @@ freeze: | $(PYTHON_DEV_ENV) pip freeze # tests ####################################################################### -pytest: | $(PYTHON_DEV_ENV) - . $(PYTHON_DEV_ENV)/bin/activate && \ - rm -rf htmlcov && \ - time tox $(args) - -ci-test: | $(PYTHON_DEV_ENV) - . $(PYTHON_DEV_ENV)/bin/activate && \ - rm -rf htmlcov && \ - time JENKINS_URL=1 tox -r $(args) - test: - ARGS=$(args) docker-compose -f tests/docker/docker-compose.yml run lona-tox + ./docker-compose run playwright tox $(args) -# linting ##################################################################### -lint: | $(PYTHON_DEV_ENV) - . $(PYTHON_DEV_ENV)/bin/activate && \ - time tox -e lint $(args) +ci-test: + ./docker-compose run playwright tox -e lint,py37,py38,py39,py310,py311 $(args) -# isort ####################################################################### -isort: | $(PYTHON_DEV_ENV) - . $(PYTHON_DEV_ENV)/bin/activate && \ - tox -e isort $(args) +lint: + ./docker-compose run playwright tox -e lint $(args) + +isort: + ./docker-compose run playwright tox -e isort $(args) # packaging ################################################################### dist: | $(PYTHON_PACKAGING_ENV) diff --git a/docker-compose b/docker-compose new file mode 100755 index 00000000..2c21f518 --- /dev/null +++ b/docker-compose @@ -0,0 +1,7 @@ +#!/bin/bash + +# set the docker user to the local user so test environments and artifacts +# can be access from inside and outside of the test container +export DOCKER_USER=$(id -u):$(id -g) + +exec docker-compose $@ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..3cea5ce7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: "3" + +networks: + net: + driver: bridge + +services: + playwright: + user: ${DOCKER_USER} + + build: + context: ./ + dockerfile: playwright.Dockerfile + + networks: + - net + + volumes: + - ./:/app + + environment: + - PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + + working_dir: "/app" + command: "tox" diff --git a/playwright.Dockerfile b/playwright.Dockerfile new file mode 100644 index 00000000..199f5f00 --- /dev/null +++ b/playwright.Dockerfile @@ -0,0 +1,30 @@ +FROM mcr.microsoft.com/playwright:v1.34.0-jammy + +# upgrade ubuntu +RUN apt update && apt upgrade -y + +# setup tzdata (required by pyenv dependencies) +RUN apt update && DEBIAN_FRONTEND=noninteractive TZ=Gmt/UTC apt-get -y install tzdata + +# install pyenv dependencies +RUN apt update && apt install -y \ + git build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ + libsqlite3-dev curl libncursesw5-dev xz-utils tk-dev libxml2-dev \ + libxmlsec1-dev libffi-dev liblzma-dev + +# setup pyenv +RUN git clone https://github.com/yyuu/pyenv.git .pyenv +ENV PYENV_ROOT $HOME/.pyenv +ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH + +RUN pyenv install 3.7:latest +RUN pyenv install 3.8:latest +RUN pyenv install 3.9:latest +RUN pyenv install 3.10:latest +RUN pyenv install 3.11:latest + +RUN pyenv global `pyenv versions --bare` + +# setup commandline tools +RUN pip3.10 install --upgrade pip tox +RUN pyenv rehash diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml deleted file mode 100644 index 414e4394..00000000 --- a/tests/docker/docker-compose.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '3' - -networks: - net: - driver: bridge - -services: - lona-tox: - build: - context: ./ - dockerfile: playwright.dockerfile - - networks: - - net - - volumes: - - ../../:/project - - .cache/envs:/project/envs - - .cache/.tox:/project/.tox - - .cache/.ms-playwright:/root/.cache/ms-playwright - - command: 'make --directory=/project pytest args="${ARGS}"' \ No newline at end of file diff --git a/tests/docker/playwright.dockerfile b/tests/docker/playwright.dockerfile deleted file mode 100644 index 78bdc789..00000000 --- a/tests/docker/playwright.dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM mcr.microsoft.com/playwright:v1.25.0-focal - -RUN apt update -RUN apt install -y make python3 python3-venv python3-pip diff --git a/tox.ini b/tox.ini index 9ab3fec5..5cf6effb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skip_missing_interpreters=True -envlist=python +envlist=py310 [tox:jenkins] @@ -8,15 +8,13 @@ envlist=lint,py37,py38,py39,py310,py311 [testenv] +passenv = PLAYWRIGHT_BROWSERS_PATH ignore_errors=True deps = -r tests/REQUIREMENTS.test.txt commands = - python -m playwright install - python -m playwright install-deps - coverage erase coverage run -a \