diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..f7546e10 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,52 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. +# ref: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions +# +name: Release + +# Always tests wheel building, but only publish to PyPI on pushed tags. +on: + pull_request: + paths-ignore: + - "docs/**" + - ".github/workflows/*.yaml" + - "!.github/workflows/release.yaml" + push: + paths-ignore: + - "docs/**" + - ".github/workflows/*.yaml" + - "!.github/workflows/release.yaml" + branches-ignore: + - "dependabot/**" + - "pre-commit-ci-update-config" + tags: ["**"] + workflow_dispatch: + +jobs: + build-release: + name: Build release + + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: install build package + run: | + pip install --upgrade pip + pip install build + pip freeze + + - name: build release + run: | + python -m build --sdist --wheel . + ls -l dist + + - name: publish to pypi + uses: pypa/gh-action-pypi-publish@release/v1 + if: startsWith(github.ref, 'refs/tags/') + with: + user: __token__ + password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 7a0ba212..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This is a GitHub workflow to auto-deploy releases on tags -# Useful references: -# https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ -# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python#publishing-to-package-registries -# https://github.community/t/how-to-run-github-actions-workflow-only-for-new-tags/16075 -# https://www.codingwithcalvin.net/git-tag-based-released-process-using-github-actions/ -name: Release - -# Always build releases (to make sure wheel-building works) -# but only publish to PyPI on tags -on: - push: - tags: "[0-9]+.[0-9]+.[0-9]+*" - branches: main - -jobs: - build-release: - name: Build release - - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install build package - run: | - python -m pip install --upgrade pip - pip install build - pip freeze - - name: Build release - run: | - python -m build --sdist --wheel . - ls -l dist - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: startsWith(github.ref, 'refs/tags/') - with: - user: __token__ - password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f16de1a..df72299a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,10 +24,7 @@ on: workflow_dispatch: env: - ETCDCTL_API: "3" PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python - ETCD_DOWNLOAD_VERSION: "3.4.24" - CONSUL_VERSION: "1.14.4" jobs: # Run "pytest tests" for various Python versions @@ -39,68 +36,108 @@ jobs: fail-fast: false matrix: include: + # Test against old and new python, and traefik v2 and v3 - python: "3.8" - backend: "etcd" - - python: "3.9" - backend: "consul" - - python: "3.10" - backend: "redis" - - python: "3.11" - steps: - # NOTE: In GitHub workflows, environment variables are set by writing - # assignment statements to a file. They will be set in the following - # steps as if would used `export MY_ENV=my-value`. - - name: Configure environment variables - run: | - echo "PATH=$PWD/bin:$PATH" >> $GITHUB_ENV + backend: file + traefik-version: "v2.10.7" + - python: "3.12" + backend: file + traefik-version: "v3.0.0-beta5" + # Test each backend that requires a python client against a modern + # version of python to ensure clients are compatible. + # + # - About redis versions: + # redis can be installed from source, as an apt package for an + # ubuntu lts release, or snap package. To avoid complexity we don't + # build from source which would allow us to pick a version to + # install, but we test against the ubuntu and snap package, where + # the ubuntu package typically is behind the snap package. + # + # redis-version is provided as a matrix input only to provide + # visibility via github's web UI. + # + - python: "3.12" + backend: redis + install-via: apt # version 6.0.16 in ubuntu 22.04, https://packages.ubuntu.com/jammy/redis + redis-version: "6.0.16?" + - python: "3.12" + backend: redis + install-via: snap # version 7.2.4 or higher, https://snapcraft.io/redis + redis-version: ">=7.2.4" + - python: "3.12" + backend: etcd + etcd-version: "v3.4.29" # https://github.com/etcd-io/etcd/releases + - python: "3.12" + backend: consul + consul-version: "1.17.2" # https://github.com/hashicorp/consul/releases (but omit v prefix) + steps: - uses: actions/checkout@v4 - # NOTE: actions/setup-python@v5 make use of a cache within the GitHub base - # environment and setup in a fraction of a second. - - name: Install Python ${{ matrix.python }} - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Install Python dependencies run: | pip install --upgrade setuptools pip - pip install -r dev-requirements.txt -e . - python -m jupyterhub_traefik_proxy.install --output=./bin + pip install -e ".[test]" + - name: List Python dependencies + run: | pip freeze + + - name: Add bin directories to PATH + run: | + echo "PATH=$PWD/bin:/snap/redis/current/usr/bin/:$PATH" >> $GITHUB_ENV + + - name: Install traefik + run: | + TRAEFIK_VERSION= + if [[ ! -z "${{ matrix.traefik-version }}" ]]; then + TRAEFIK_VERSION=--traefik-version=${{ matrix.traefik-version }} + fi + + python -m jupyterhub_traefik_proxy.install --output=./bin ${TRAEFIK_VERSION} + + traefik version + - name: Install consul if: matrix.backend == 'consul' run: | - curl -L https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip > consul.zip + curl -L https://releases.hashicorp.com/consul/${{ matrix.consul-version }}/consul_${{ matrix.consul-version }}_linux_amd64.zip > consul.zip unzip consul.zip -d ./bin consul + consul version + - name: Install etcd if: matrix.backend == 'etcd' run: | - curl -L https://github.com/etcd-io/etcd/releases/download/v${ETCD_DOWNLOAD_VERSION}/etcd-v${ETCD_DOWNLOAD_VERSION}-linux-amd64.tar.gz > etcd.tar.gz + curl -L https://github.com/etcd-io/etcd/releases/download/${{ matrix.etcd-version }}/etcd-${{ matrix.etcd-version }}-linux-amd64.tar.gz > etcd.tar.gz tar -xzf etcd.tar.gz -C ./bin --strip-components=1 --wildcards '*/etcd*' - - name: Install redis - if: matrix.backend == 'redis' - run: | - sudo apt-get -y install redis-server + etcdctl version - - name: Select tests + - name: Install redis via ${{ matrix.install-via }} + if: matrix.backend == 'redis' run: | - if [[ ! -z "${{ matrix.backend }}" ]]; then - # select backend subset - echo "PYTEST_ADDOPTS=-k ${{ matrix.backend }}" >> "${GITHUB_ENV}" + if [[ "${{ matrix.install-via }}" == "apt" ]]; then + sudo apt-get -y install redis else - # default: select everything _but_ the KV backend tests - echo "PYTEST_ADDOPTS=-k 'not etcd and not consul and not redis'" >> "${GITHUB_ENV}" + sudo snap install redis fi + redis-server --version + - name: Run tests run: | - # Using the "--slow-last" flag, should run the slow tests last - pytest -v --durations 10 --maxfail 3 --slow-last --color=yes --cov=jupyterhub_traefik_proxy tests + if [[ "${{ matrix.backend }}" == "file" ]]; then + # select everything _but_ the KV store backend tests + export PYTEST_ADDOPTS="-k 'not etcd and not consul and not redis'" + else + # select backend subset + export PYTEST_ADDOPTS="-k ${{ matrix.backend }}" + fi - - name: Submit codecov report - run: | - codecov + pytest + + - uses: codecov/codecov-action@v3 diff --git a/readthedocs.yml b/.readthedocs.yaml similarity index 92% rename from readthedocs.yml rename to .readthedocs.yaml index 48bcd406..d8cfa774 100644 --- a/readthedocs.yml +++ b/.readthedocs.yaml @@ -16,7 +16,3 @@ python: - method: pip path: . - requirements: docs/requirements.txt - -formats: - - htmlzip - - epub diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d32757a9..c97aeb94 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,14 +10,10 @@ This package requires Python >= 3.6. As a Python package, you can set up a development environment by cloning this repo and running: - python3 -m pip install --editable . + python3 -m pip install --editable ".[test]" from the repo directory. -You can also install the tools we use for testing and development with: - - python3 -m pip install -r dev-requirements.txt - ### Auto-format with pre-commit We use the [pre-commit](https://pre-commit.com) tool for autoformatting. @@ -35,6 +31,6 @@ If it makes any changes, it'll let you know and you can make the commit again wi After doing a development install, you can run the tests with: - pytest -v + pytest in the repo directory. diff --git a/README.md b/README.md index 0af1d00e..d517f6d1 100644 --- a/README.md +++ b/README.md @@ -37,22 +37,22 @@ with examples for the three different implementations. ## Running tests -There are some tests that use _etcdctl_ command line client for etcd. Make sure -to set environment variable `ETCDCTL_API=3` before running the tests, so that -the v3 API to be used, e.g.: +You can then run the all the test suite from the _traefik-proxy_ directory with: ``` -$ export ETCDCTL_API=3 +$ pytest ``` -You can then run the all the test suite from the _traefik-proxy_ directory with: +Or you can run a specific test file with: ``` -$ pytest -v ./tests +$ pytest tests/ ``` -Or you can run a specific test file with: +There are some tests that use _etcdctl_ command line client for etcd. Make sure +to set environment variable `ETCDCTL_API=3` before running the tests if etcd +version 3.3 or older is used, so that the v3 API to be used, e.g.: ``` -$ pytest -v ./tests/ +$ export ETCDCTL_API=3 ``` diff --git a/RELEASE.md b/RELEASE.md index f8171496..0981782c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,26 +1,22 @@ # How to make a release -`traefik-proxy` is a package [available on -PyPI](https://pypi.org/project/jupyterhub-traefik-proxy/). These are -instructions on how to make a release on PyPI. +`jupyterhub-traefik-proxy` is a package available on [PyPI]. -For you to follow along according to these instructions, you need: +These are the instructions on how to make a release. -- To have push rights to the [jupyterhub-traefik-proxy GitHub - repository](https://github.com/jupyterhub/traefik-proxy). +## Pre-requisites + +- Push rights to this GitHub repository ## Steps to make a release 1. Create a PR updating `docs/source/changelog.md` with [github-activity] and - continue only when its merged. - - ```shell - pip install github-activity + continue when its merged. For details about this, see the [team-compass + documentation] about it. - github-activity --heading-level=3 jupyterhub/traefik-proxy - ``` + [team-compass documentation]: https://jupyterhub-team-compass.readthedocs.io/en/latest/practices/releases.html -1. Checkout main and make sure it is up to date. +2. Checkout main and make sure it is up to date. ```shell git checkout main @@ -28,22 +24,30 @@ For you to follow along according to these instructions, you need: git reset --hard origin/main ``` -1. Update the version, make commits, and push a git tag with `tbump`. +3. Update the version, make commits, and push a git tag with `tbump`. ```shell pip install tbump - tbump --dry-run ${VERSION} + ``` + + `tbump` will ask for confirmation before doing anything. + ```shell + # Example versions to set: 1.0.0, 1.0.0b1 + VERSION= tbump ${VERSION} ``` Following this, the [CI system] will build and publish a release. -1. Reset the version back to dev, e.g. `2.1.0.dev` after releasing `2.0.0` +4. Reset the version back to dev, e.g. `1.0.1.dev` after releasing `1.0.0`. ```shell + # Example version to set: 1.0.1.dev + NEXT_VERSION= tbump --no-tag ${NEXT_VERSION}.dev ``` -[ci system]: https://github.com/jupyterhub/traefik-proxy/actions [github-activity]: https://github.com/executablebooks/github-activity +[pypi]: https://pypi.org/project/jupyterhub-traefik-proxy/ +[ci system]: https://github.com/jupyterhub/traefik-proxy/actions/workflows/release.yaml diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index ddd8c21d..00000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -black -certipy -codecov -# add backend packages -jupyterhub-traefik-proxy[redis,etcd,consul] -notebook>=4.0 -pytest -# FIXME: unpin pytest-asyncio -pytest-asyncio>=0.17,<0.23 -pytest-cov -redis -websockets diff --git a/docs/source/details.md b/docs/source/details.md index f021883b..87797a95 100644 --- a/docs/source/details.md +++ b/docs/source/details.md @@ -8,7 +8,7 @@ Because of **security** concerns, in traefik-proxy implementation, traefik api e The port on which traefik-proxy's api will run, as well as the username and password used for authenticating, can be passed to the proxy through `jupyterhub_config.py`, e.g.: -``` +```python c.TraefikFileProviderProxy.traefik_api_url = "http://127.0.0.1:8099" c.TraefikFileProviderProxy.traefik_api_password = "admin" c.TraefikFileProviderProxy.traefik_api_username = "admin" @@ -77,12 +77,12 @@ These classes only need to implement: You can then run the all the test suite from the _traefik-proxy_ directory with: -``` -$ pytest -v ./tests +```shell +pytest ``` Or you can run a specific test with: -``` -$ pytest -v ./tests/ +```shell +pytest tests/ ``` diff --git a/docs/source/install.md b/docs/source/install.md index 8d97ba3b..08c08f3b 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -63,7 +63,7 @@ If you want to use a key-value store to mediate configuration (mainly for use in distributed deployments, such as containers), you can get the key-value stores via their respective installation pages: -- Install [`redis-server`](https://redis.io/docs/install/install-redis/) +- Install [`redis`](https://redis.io/docs/install/install-redis/) - Install [`etcd`](https://github.com/etcd-io/etcd/releases) diff --git a/docs/source/redis.md b/docs/source/redis.md index 0d06921b..230afe36 100644 --- a/docs/source/redis.md +++ b/docs/source/redis.md @@ -189,7 +189,11 @@ This is an example setup for using JupyterHub and TraefikRedisProxy managed by a 2. Start a single-node Redis cluster on the default port on localhost. e.g.: ```bash - $ redis-server + redis-server + + # if redis-server isn't found on path, and you installed it with snap, you + # probably need to do this: + # export PATH="/snap/redis/current/usr/bin/:$PATH" ``` 3. Create a traefik static configuration file, _traefik.toml_, e.g:. diff --git a/jupyterhub_traefik_proxy/redis.py b/jupyterhub_traefik_proxy/redis.py index cfe6b24b..83251694 100644 --- a/jupyterhub_traefik_proxy/redis.py +++ b/jupyterhub_traefik_proxy/redis.py @@ -148,10 +148,7 @@ async def _kv_atomic_delete(self, *keys): await asyncio.gather(*futures) async def _kv_get_tree(self, prefix): - """Return all data under prefix as a dict - - Should probably use `unflatten_dict_from_kv` - """ + """Return all data under prefix as a dict""" if not prefix.endswith(self.kv_separator): prefix = prefix + self.kv_separator diff --git a/pyproject.toml b/pyproject.toml index 2ce4ad74..ad4e2131 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,10 +50,9 @@ profile = "black" # ref: https://docs.pytest.org/en/stable/ # [tool.pytest.ini_options] -addopts = "--verbose --color=yes --durations=10" +addopts = "--verbose --color=yes --durations=10 --maxfail 3 --slow-last --cov=jupyterhub_traefik_proxy" asyncio_mode = "auto" -# Ignore thousands of tests in dependencies installed in a virtual environment -norecursedirs = "lib lib64" +testpaths = ["tests"] [tool.tbump] github_url = "https://github.com/jupyterhub/traefik-proxy" diff --git a/setup.py b/setup.py index 9bd23189..58da1d79 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,15 @@ # see https://github.com/jupyterhub/traefik-proxy/issues/155 for more "consul": ["python-consul2"], "etcd": ["etcdpy"], + "test": [ + "jupyterhub-traefik-proxy[redis,etcd,consul]", + "certipy", + "notebook>=4.0", + "pytest", + "pytest-asyncio>=0.17,<0.23", # FIXME: unpin pytest-asyncio + "pytest-cov", + "websockets", + ], }, python_requires=">=3.8", author="Project Jupyter Contributors",