From 8ba5c3beef9682ae7680d05d34c6cacad76d2182 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Tue, 20 Jun 2023 12:19:32 -0400 Subject: [PATCH] test: use MySQL when running unit tests in CI This will run all CI tests against MySQL 8.0. It changes the tox py38-django{32|42} envs to use a new MySQL settings file. Running pytest manually will continue to use the in-memory SQLite database. --- .github/workflows/ci.yml | 23 ++++++++++++++++++++++- mysql_test_settings.py | 26 ++++++++++++++++++++++++++ requirements/base.txt | 4 +++- requirements/ci.txt | 8 ++++---- requirements/dev.txt | 39 +++++++++++++++++++++------------------ requirements/doc.txt | 23 +++++++++++++---------- requirements/quality.txt | 33 ++++++++++++++++++--------------- requirements/test.in | 2 ++ requirements/test.txt | 16 ++++++++++------ tox.ini | 4 ++++ 10 files changed, 123 insertions(+), 55 deletions(-) create mode 100644 mysql_test_settings.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eff1b8e..43211145 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,30 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] # Add macos-latest later? python-version: ['3.8'] toxenv: ["py38-django32", "py38-django42"] + # We're only testing against MySQL 8 right now because 5.7 is + # incompatible with Djagno 4.2. We'd have to make the tox.ini file more + # complicated than it's worth given the short expected shelf-life of + # MySQL 5.7 in our stack. + mysql-version: ["8"] + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "test_oel_db" + MYSQL_USER: "test_oel_user" + MYSQL_PASSWORD: "test_oel_pass" + MYSQL_RANDOM_ROOT_PASSWORD: true + # these options are blatantly copied from edx-platform's values + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: - uses: actions/checkout@v3 - name: setup python diff --git a/mysql_test_settings.py b/mysql_test_settings.py new file mode 100644 index 00000000..ae8e37d3 --- /dev/null +++ b/mysql_test_settings.py @@ -0,0 +1,26 @@ +""" +This is an extension of the default test_settings.py file that uses MySQL for +the backend. While the openedx-learning apps should run fine using SQLite, they +also do some MySQL-specific things around charset/collation settings and row +compression. + +The tox targets for py38-django32 and py38-django42 will use this settings file. +For the most part, you can use test_settings.py instead (that's the default if +you just run "pytest" with no arguments). +""" + +from test_settings import * + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": "oel_db", + "USER": "test_oel_user", + "PASSWORD": "test_oel_pass", + "HOST": "127.0.0.1", + "PORT": "3306", + "OPTIONS": { + "charset": "utf8mb4" + } + } +} diff --git a/requirements/base.txt b/requirements/base.txt index d4893446..97ec0354 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via django django==3.2.19 # via @@ -19,3 +19,5 @@ pytz==2023.3 # djangorestframework sqlparse==0.4.4 # via django +typing-extensions==4.6.3 + # via asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index d851c07e..6e1dd7f9 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,7 +8,7 @@ click==8.1.3 # via import-linter distlib==0.3.6 # via virtualenv -filelock==3.12.0 +filelock==3.12.2 # via # tox # virtualenv @@ -18,7 +18,7 @@ import-linter==1.9.0 # via -r requirements/ci.in packaging==23.1 # via tox -platformdirs==3.5.1 +platformdirs==3.6.0 # via virtualenv pluggy==1.0.0 # via tox @@ -34,9 +34,9 @@ tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # grimp # import-linter -virtualenv==20.23.0 +virtualenv==20.23.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 9f6bd533..b2dcdaf2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/quality.txt # django @@ -49,11 +49,11 @@ code-annotations==1.3.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/quality.txt # pytest-cov -diff-cover==7.5.0 +diff-cover==7.6.0 # via -r requirements/dev.in dill==0.3.6 # via @@ -86,7 +86,7 @@ exceptiongroup==1.1.1 # via # -r requirements/quality.txt # pytest -filelock==3.12.0 +filelock==3.12.2 # via # -r requirements/ci.txt # tox @@ -104,7 +104,7 @@ import-linter==1.9.0 # via # -r requirements/ci.txt # -r requirements/quality.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # -r requirements/quality.txt # keyring @@ -130,7 +130,7 @@ jinja2==3.1.2 # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.13.1 +keyring==24.0.0 # via # -r requirements/quality.txt # twine @@ -138,11 +138,11 @@ lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt # astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via # -r requirements/quality.txt # rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/quality.txt # jinja2 @@ -158,6 +158,8 @@ more-itertools==9.1.0 # via # -r requirements/quality.txt # jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/quality.txt packaging==23.1 # via # -r requirements/ci.txt @@ -178,7 +180,7 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -222,7 +224,7 @@ pylint-django==2.5.3 # via # -r requirements/quality.txt # edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # -r requirements/quality.txt # pylint-celery @@ -231,12 +233,12 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/quality.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/quality.txt pytest-django==4.5.2 # via -r requirements/quality.txt @@ -254,11 +256,11 @@ pyyaml==6.0 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==37.3 +readme-renderer==40.0 # via # -r requirements/quality.txt # twine -requests==2.30.0 +requests==2.31.0 # via # -r requirements/quality.txt # requests-toolbelt @@ -271,7 +273,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.3.5 +rich==13.4.2 # via # -r requirements/quality.txt # twine @@ -324,21 +326,22 @@ tox-battery==0.6.1 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/quality.txt -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/ci.txt # -r requirements/quality.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # -r requirements/quality.txt # requests # twine -virtualenv==20.23.0 +virtualenv==20.23.1 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index aa92d8e5..d475be06 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -66,7 +66,7 @@ imagesize==1.4.1 # via sphinx import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via sphinx iniconfig==2.0.0 # via @@ -77,10 +77,12 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,12 +108,12 @@ pygments==2.15.1 # pydata-sphinx-theme # readme-renderer # sphinx -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -129,9 +131,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via -r requirements/doc.in -requests==2.30.0 +requests==2.31.0 # via sphinx restructuredtext-lint==1.4.0 # via doc8 @@ -183,13 +185,14 @@ tomli==2.0.1 # doc8 # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # grimp # import-linter # pydata-sphinx-theme -urllib3==2.0.2 +urllib3==2.0.3 # via requests webencodings==0.5.1 # via bleach diff --git a/requirements/quality.txt b/requirements/quality.txt index b36f07ab..ab8f3977 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ code-annotations==1.3.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -60,7 +60,7 @@ idna==3.4 # via requests import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # keyring # twine @@ -80,13 +80,13 @@ jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.13.1 +keyring==24.0.0 # via twine lazy-object-proxy==1.9.0 # via astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 @@ -96,6 +96,8 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==9.1.0 # via jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,7 +108,7 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via pylint pluggy==1.0.0 # via @@ -130,16 +132,16 @@ pylint-celery==0.3 # via edx-lint pylint-django==2.5.3 # via edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -156,9 +158,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via twine -requests==2.30.0 +requests==2.31.0 # via # requests-toolbelt # twine @@ -166,7 +168,7 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.3.5 +rich==13.4.2 # via twine six==1.16.0 # via @@ -197,15 +199,16 @@ tomlkit==0.11.8 # via pylint twine==4.0.2 # via -r requirements/quality.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # requests # twine diff --git a/requirements/test.in b/requirements/test.in index 65ac63e8..48446899 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -3,6 +3,8 @@ -r base.txt # Core dependencies for this package +mysqlclient<3.0 # MySQL support + coverage # Code coverage reporting import-linter # Track our internal dependencies diff --git a/requirements/test.txt b/requirements/test.txt index ae4cfa04..8865d16b 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/base.txt # django @@ -14,7 +14,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.in # pytest-cov @@ -34,20 +34,22 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.in packaging==23.1 # via pytest pbr==5.11.1 # via stevedore pluggy==1.0.0 # via pytest -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.in # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in @@ -73,7 +75,9 @@ tomli==2.0.1 # coverage # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via + # -r requirements/base.txt + # asgiref # grimp # import-linter diff --git a/tox.ini b/tox.ini index 36271841..23e2650a 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,10 @@ deps = django32: Django>=3.2,<3.3 django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt +setenv = + # Note that the django32/django42 targets use MySQL, but running pytest by + # itself (without tox) will run tests in SQLite for developer convenience. + DJANGO_SETTINGS_MODULE = mysql_test_settings commands = pytest {posargs}