diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..0af041f89 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.devcontainer +.github +.ropeproject +.vscode +build +dist +docker/alpine/var +docker/elastic +docker/solr +docker/ubuntu/var +docker/zeo/var +docker/zope/var diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3a996cd83..ff9124beb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,48 +2,49 @@ name: tests on: - push: - pull_request: - schedule: - - cron: '0 12 * * 0' # run once a week on Sunday - # Allow to run this workflow manually from the Actions tab - workflow_dispatch: + push: + pull_request: + schedule: + - cron: "0 12 * * 0" # run once a week on Sunday + # Allow to run this workflow manually from the Actions tab + workflow_dispatch: jobs: - build: - strategy: - # We want to see all failures: - fail-fast: false - matrix: - os: - - ubuntu - config: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" - runs-on: ${{ matrix.os }}-latest - name: ${{ matrix.os }}-${{ matrix.config }} - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.config }} - - name: Pip cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.config }}-${{ hashFiles('setup.*', 'requirements*') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.config }}- - ${{ runner.os }}-pip- - - name: Install dependencies - run: | - python -m pip install --upgrade pip pytest - pip install -e . - - name: Test - run: | - tree -L 1 - pytest ./tests/ + build: + strategy: + # We want to see all failures: + fail-fast: false + matrix: + os: + - ubuntu + config: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + runs-on: ${{ matrix.os }}-latest + name: ${{ matrix.os }}-${{ matrix.config }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.config }} + - name: Pip cache + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.config }}-${{ hashFiles('setup.*', 'requirements*') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.config }}- + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + python -m pip install --upgrade pip pytest + pip install -e . + - name: Test + run: | + tree -L 1 + pytest ./tests/ diff --git a/.vscode/launch.json b/.vscode/launch.json index 5fa11c32d..c3496f6b0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,37 +1,38 @@ { "version": "0.2.0", - "configurations": [ { "name": "ZMS5-DEV", "type": "python", "request": "launch", - "program": "~/vpy38/bin/runwsgi", + "program": "~/vpy313/bin/runwsgi", "justMyCode": false, "console": "integratedTerminal", "args": [ "--debug", "--verbose", "~/instance/zms5_dev/etc/zope.ini", - "debug-mode=on", - // "http_port=8086", + "debug-mode=on" ], "env": { - "PYTHONUNBUFFERED":"1", + "PYTHONUNBUFFERED": "1", "CONFIG_FILE": "~/instance/zms5_dev/etc/zope.ini", "INSTANCE_HOME": "~/instance/zms5_dev", "CLIENT_HOME": "~/instance/zms5_dev", - "PYTHON": "~/vpy38/bin/python", - "SOFTWARE_HOME": "~/vpy38/bin/" + "PYTHON": "~/vpy313/bin/python", + "SOFTWARE_HOME": "~/vpy313/bin/" }, - "serverReadyAction":{ - "pattern":"Serving on http://127.0.0.1:8081", + "serverReadyAction": { + "pattern": "Serving on http://127.0.0.1:8081", "uriFormat": "http://127.0.0.1:8081/manage_main", - "action": "openExternally", - }, + "action": "openExternally" + } }, - - - + { + "name": "Python Debugger: Python File", + "type": "debugpy", + "request": "launch", + "program": "${file}" + } ] } \ No newline at end of file diff --git a/Products/zms/_fileutil.py b/Products/zms/_fileutil.py index b13c37fe1..13bda40b3 100644 --- a/Products/zms/_fileutil.py +++ b/Products/zms/_fileutil.py @@ -419,6 +419,8 @@ def extractZipArchive(file): zf = zipfile.ZipFile( file, 'r') for name in zf.namelist(): + if name.startswith('__MACOSX/') or name.endswith('.DS_Store'): + continue dir = getOSPath( name) i = dir.rfind( os.sep) if i > 0: @@ -516,4 +518,4 @@ def tail_lines(filename,linesback=10,returnlist=0): for l in lines[start:len(lines)-1]: out=out + l + "\n" return out -################################################################################ \ No newline at end of file +################################################################################ diff --git a/Products/zms/repositoryutil.py b/Products/zms/repositoryutil.py index e88feda70..01d5e635e 100644 --- a/Products/zms/repositoryutil.py +++ b/Products/zms/repositoryutil.py @@ -30,6 +30,7 @@ import inspect import os import re +import sys # Product Imports. from Products.zms import IZMSConfigurationProvider from Products.zms import IZMSRepositoryProvider @@ -65,8 +66,13 @@ def get_repo_providers(context): """ def get_class(py): id = re.findall('class (.*?):', py)[0] - exec(py) - return eval(id) + if sys.version_info >= (3, 13): + py = py + "\nglobal c\nc = " + id + exec(py, globals=globals(), locals=locals()) + return eval("c", globals=globals(), locals=locals()) + else: + exec(py) + return eval(id) """ diff --git a/bin/run_tests_in_docker b/bin/run_tests_in_docker new file mode 100755 index 000000000..9eb48d489 --- /dev/null +++ b/bin/run_tests_in_docker @@ -0,0 +1,5 @@ +#!/bin/sh + +exec docker compose run --rm zope \ + watching_testrunner --basepath Products --basepath tests \ + -- pytest --tb=short $@ diff --git a/docker-compose.yml b/docker-compose.yml index 195d92bb5..b77d04c34 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,56 @@ services: - zope: - build: ./docker/base - image: zope:latest - depends_on: - - zeo - stop_grace_period: 1s # SIGKILL after 1s, as zope is always taking the full 10 seconds - ports: - - 80:80 - volumes: - - .:/home/zope/venv/src/zms/ - - ./docker/zope/etc/:/home/zope/etc/ - - ./docker/zope/var/:/home/zope/var/ - # To share external methods between Zope/ZEO/Docker clients folder needs to be mounted - - ./docker/zope/Extensions/:/home/zope/Extensions/ - # allow attaching to the container to debug with `breakpoint()` - stdin_open: true - tty: true + zope: + build: + context: . + dockerfile: ./docker/base/Dockerfile + image: localhost/zms:latest + depends_on: + - zeo + stop_grace_period: 1s # SIGKILL after 1s, as zope is always taking the full 10 seconds + command: runwsgi --debug --verbose etc/zope.ini debug-mode=on http_port=80 + ports: + - 80:80 + volumes: + - ./docker/zope/etc/:/home/zope/etc/ + - ./docker/zope/var/:/home/zope/var/ + # To share external methods between Zope/ZEO/Docker clients folder needs to be mounted + - ./docker/zope/Extensions/:/home/zope/Extensions/ + # source code + - ./tests:/home/zope/tests + - ./test_output:/home/zope/test_output + - ./selenium_tests:/home/zope/selenium_tests + develop: + watch: + # sync+restart + - action: sync+restart + path: ./Products + target: /home/zope/Products + # rebuild + - action: rebuild + path: docker/base/Dockerfile + - action: rebuild + path: requirements.txt + - action: rebuild + path: requirements-zeo.txt + - action: rebuild + path: requirements-full.txt + - action: rebuild + path: requirements-dev.txt + - action: rebuild + path: setup.py + - action: rebuild + path: setup.cfg - zeo: - image: zope:latest - command: runzeo --configure etc/zeo.conf - volumes: - - ./docker/zeo/etc/:/home/zope/etc/ - - ./docker/zeo/var/:/home/zope/var/ + # allow attaching to the container to debug with `breakpoint()` + stdin_open: true + tty: true + + zeo: + build: + context: . + dockerfile: ./docker/base/Dockerfile + image: localhost/zms:latest + command: runzeo --configure etc/zeo.conf + volumes: + - ./docker/zeo/etc/:/home/zope/etc/ + - ./docker/zeo/var/:/home/zope/var/ diff --git a/docker/TODO.md b/docker/TODO.md index 42e6c1691..9e40b314b 100755 --- a/docker/TODO.md +++ b/docker/TODO.md @@ -4,12 +4,12 @@ - [X] Starting the bare docker file will give you a basic zope / zms - [x] everything as similar to our server deployment as possible to allow easy migration - [x] modern os and python -- [ ] simple to use and develop in vscode -> .devcontainer! +- [x] simple to use and develop in vscode -> .devcontainer! FH has different solution - [x] all mutable data in mounted volumes -- [ ] example systemd files to run everything - - [ ] this should show how automated container updates are done! -- [ ] example nginx config so you get the same experience as on the server - [x] Allow working on zms inside the container +- [x] example systemd files to run everything + - [x] this should show how automated container updates are done! +- [ ] example nginx config so you get the same experience as on the server - [ ] Full development experience with all dependennt services locally (mariadb, memcached, …) # TODOs @@ -17,5 +17,8 @@ - [x] Create basic Dockerfile for the project - [x] specialize them for zeoserver and zope - [x] create docker-compose file that runs each server separately -- [ ] add devcontainer.json to develop and run everything from vscode -- [ ] mount the zms source live into the container so working within it becomes possible +- [x] add devcontainer.json to develop and run everything from vscode +- [x] mount the zms source live into the container so working within it becomes possible +- [x] remove debug mode from zope Dockerfile +- [x] add script to run tests in docker +- [ ] add nginx, mariadb, memcached to docker-compose for a fully featured development environment, that can run production like configs diff --git a/docker/alpine/Extensions/readme.md b/docker/alpine/Extensions/readme.md index 8f483b4f4..3cddeefe7 100755 --- a/docker/alpine/Extensions/readme.md +++ b/docker/alpine/Extensions/readme.md @@ -1,4 +1,13 @@ # Externalizing Extensions for Docker Hint: Mounting the folder ./Extensions keeps the external functions -synchronous to all all ZEO clients and Docker containers. \ No newline at end of file +synchronous to all all ZEO clients and Docker containers. + +Hint: if the docker container cannot write to the ./Extensions or ./var folder, +on a dev system you can simply set the permissions to 777. +Important:this not recommended for production! + +```bash +chmod -R 777 Extensions +chmod -R 777 var +``` diff --git a/docker/alpine/docker-compose.yml b/docker/alpine/docker-compose.yml index ad64eabac..8965b0a94 100755 --- a/docker/alpine/docker-compose.yml +++ b/docker/alpine/docker-compose.yml @@ -1,8 +1,8 @@ version: "3.7" services: - zms5: + zms5-alpine: build: . - image: zms5:latest + image: zms5-alpine:latest ports: - 8085:8085 - 8086:8086 diff --git a/docker/alpine/dockerfile b/docker/alpine/dockerfile index fccedd17d..077345347 100755 --- a/docker/alpine/dockerfile +++ b/docker/alpine/dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM python:3.13-alpine EXPOSE 8085 EXPOSE 8086 @@ -18,7 +18,11 @@ ENV VIRTUAL_ENV=/home/zope/venv RUN python3 -m venv venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip install -U pip wheel setuptools -RUN pip install -U -e git+https://github.com/zms-publishing/ZMS.git@main#egg=ZMS +# RUN pip install -U -e git+https://github.com/zms-publishing/ZMS.git@main#egg=ZMS +# Install Zope --editable first, so that the toml-setup can be used +# https://github.com/zms-publishing/ZMS/pull/302#issuecomment-2327312673 +RUN pip install -e git+https://github.com/zopefoundation/Zope.git@master#egg=Zope +RUN pip install -e git+https://github.com/zms-publishing/ZMS.git@fb_pip_using_pyproject_toml#egg=ZMS RUN pip install -r https://raw.githubusercontent.com/zms-publishing/ZMS5/master/requirements-full.txt RUN pip install ZEO RUN pip install itsdangerous diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 20c54e48a..072919890 100755 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -27,12 +27,12 @@ EOR # Drop root privileges ENV UID=1000 RUN useradd --system --create-home --uid $UID zope -USER zope:zope +USER zope WORKDIR /home/zope/ # Install uv -COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv -ENV UV_LINK_MODE=copy +# COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv +# ENV UV_LINK_MODE=copy # Create venv and permanently enable it # ARG CREATE_VENV_COMMAND="uv venv" @@ -42,9 +42,18 @@ RUN $CREATE_VENV_COMMAND $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" # Enable caching of pip packages to speed up rebuild times -# cannot use uv as long as so many editable packages need to be installed +# uv follows the new editable spec, even if the package (zms) does not provide a pyproject.toml +# this creates entries like __editable__.ZMS-5.2.0.finder.__path_hook__ in `Products.__path__` +# which Zope cannot handle yet +# @see https://github.com/zopefoundation/Zope/issues/1239 +# uv also cannot install editable eggs from git repos, but I think we don't need that # ARG PIP_INSTALL="uv pip install" +# may need to switch back to pip if the pyproject.toml requirement kicks in ARG PIP_INSTALL="pip install --config-settings editable_mode=compat" + +COPY --chown=zope:zope Products/zms/version.txt Products/zms/version.txt +COPY --chown=zope:zope requirements*.txt setup.py setup.cfg README.rst ./ + ENV PIP_CACHE_DIR=/home/zope/venv/cache RUN --mount=type=cache,uid=$UID,target=$PIP_CACHE_DIR,sharing=locked < -# TODO talk to FH if this makes sense. Might require a slightly different workflow from them. -# Currently always enabled on our servers. - -# Regarding --debug -# @see https://zope.readthedocs.io/en/latest/operation.html#running-zope-in-the-foreground -# seems to sugges that this also enables the debug-mode directive -# TODO talk to FH to allow me to check that - -exec $venv_bin_dir/runwsgi --debug --verbose $instance_dir/etc/zope.ini debug-mode=on http_port=80 diff --git a/docker/systemd/zms-restart.service b/docker/systemd/zms-restart.service new file mode 100644 index 000000000..9bd909726 --- /dev/null +++ b/docker/systemd/zms-restart.service @@ -0,0 +1,27 @@ +[Unit] +Description=ZMS/Zope rebuilder and restarter + +After=docker.service +Requires=docker.service + + +[Service] +#User=zope +#Group=zope +Restart=always + +## FIXME /path/to/oidc_client/ needs to point to the root folder of the source code of the oidc_client +## so docker can actually build the image +## TODO probably best to point to a rebuild script? +Environment="INSTANCE_HOME=/path/to/instance_home" +Environment="DOCKER_IMAGE=localhost/zms DOCKER_TAG=latest" + +TimeoutStartSec=0 +PrivateTmp=true + +ExecStartPre=/usr/bin/docker build --no-cache --pull --tag $DOCKER_IMAGE:$DOCKER_TAG $INSTANCE_HOME +ExecStart=/usr/bin/systemd restart zms@*.service + +[Install] +WantedBy=multi-user.target +WantedBy=zms.target diff --git a/docker/systemd/zms-restart.timer b/docker/systemd/zms-restart.timer new file mode 100644 index 000000000..8abf36567 --- /dev/null +++ b/docker/systemd/zms-restart.timer @@ -0,0 +1,15 @@ +[Unit] +Description=Timer for ZMS/Zope to restart and pull os updates +# Propagate start and stop from target +PartOf=zms.target +# Start after target +After=zms.target + +[Timer] +OnBootSec=10m +RandomizedDelaySec=1h +OnCalendar=daily + +[Install] +WantedBy=timers.target +WantedBy=zms.target diff --git a/docker/systemd/zms-zeopack.service b/docker/systemd/zms-zeopack.service new file mode 100644 index 000000000..c61714971 --- /dev/null +++ b/docker/systemd/zms-zeopack.service @@ -0,0 +1,10 @@ +[Unit] +Description=ZMS/Zope ZEO zeopack +After=network-online.target + +[Service] +Type=simple +User=zope +# I do not know yet how to execute this correctly +# may require a script that execs into the zeo container to trigger this? +ExecStart=/home/zope/instance/bin/zeopack diff --git a/docker/systemd/zms-zeopack.timer b/docker/systemd/zms-zeopack.timer new file mode 100644 index 000000000..8abf36567 --- /dev/null +++ b/docker/systemd/zms-zeopack.timer @@ -0,0 +1,15 @@ +[Unit] +Description=Timer for ZMS/Zope to restart and pull os updates +# Propagate start and stop from target +PartOf=zms.target +# Start after target +After=zms.target + +[Timer] +OnBootSec=10m +RandomizedDelaySec=1h +OnCalendar=daily + +[Install] +WantedBy=timers.target +WantedBy=zms.target diff --git a/docker/systemd/zms.target b/docker/systemd/zms.target new file mode 100644 index 000000000..63f5eb7ee --- /dev/null +++ b/docker/systemd/zms.target @@ -0,0 +1,6 @@ +[Unit] +Description=ZMS/Zope and all of its dependencies +After=network-online.target + +[Install] +WantedBy=multi-user.target diff --git a/docker/systemd/zms@.service b/docker/systemd/zms@.service new file mode 100644 index 000000000..aeae6fb3f --- /dev/null +++ b/docker/systemd/zms@.service @@ -0,0 +1,30 @@ +[Unit] +Description=ZMS/Zope + +After=docker.service +Requires=docker.service + +Requires=memcached.service +After=memcached.service + +Requires=mariadb.service +After=mariadb.service + + +[Service] +#User=zope +#Group=zope +Restart=always + +Environment="ZOPE_PUBLIC_PORT=%i" +Environment="DOCKER_IMAGE=localhost/zms DOCKER_TAG=latest" + +TimeoutStartSec=0 +PrivateTmp=true + +ExecStartPre=-/usr/bin/docker stop $DOCKER_IMAGE +ExecStart=/usr/bin/docker run --rm --publish 127.0.0.1:$ZOPE_PUBLIC_PORT:80 --name $DOCKER_IMAGE $DOCKER_IMAGE:$DOCKER_TAG + +[Install] +WantedBy=multi-user.target +WantedBy=zms.target diff --git a/docker/ubuntu/Extensions/readme.md b/docker/ubuntu/Extensions/readme.md index 820867944..4ed7ecf43 100755 --- a/docker/ubuntu/Extensions/readme.md +++ b/docker/ubuntu/Extensions/readme.md @@ -1 +1,13 @@ -# Externalizing Extensions for Docker \ No newline at end of file +# Externalizing Extensions for Docker + +Hint: Mounting the folder ./Extensions keeps the external functions +synchronous to all all ZEO clients and Docker containers. + +Hint: if the docker container cannot write to the ./Extensions or ./var folder, +on a dev system you can simply set the permissions to 777. +Important:this not recommended for production! + +```bash +chmod -R 777 Extensions +chmod -R 777 var +``` \ No newline at end of file diff --git a/requirements-full.txt b/requirements-full.txt index 8a320d6c2..b1d7da163 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -2,7 +2,7 @@ -r requirements.txt # Theming --e git+https://github.com/zms-publishing/Products.zms-skins.git#egg=Products.zms-skins +git+https://github.com/zms-publishing/Products.zms-skins.git#egg=Products.zms-skins # SQL SQLAlchemy==1.4.49 @@ -32,7 +32,7 @@ asyncio # PAS Products.PluggableAuthService Products.PluginRegistry -# -e git+https://github.com/sntl-projects/Products.zmsPluggableAuthService.git#egg=Products.zmsPluggableAuthService +# git+https://github.com/sntl-projects/Products.zmsPluggableAuthService.git#egg=Products.zmsPluggableAuthService # MemCacheD to replace mappingstorage -Products.mcdutils>=3.2 \ No newline at end of file +Products.mcdutils>=3.2 diff --git a/setup.py b/setup.py index e44fe094b..524eaee09 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -The ZMS environment consists of an application server based on pegged versions +The ZMS environment consists of an application server based on pegged versions of depending packages (see requirements.txt). Use @@ -69,5 +69,14 @@ def read_version(): zip_safe = False, extras_require = { 'zeo' : open(os.path.join(setup_path, 'requirements-zeo.txt')).readlines(), + 'dev' : [ + # allow debugging from vscode + 'debugpy', + # test execution in container + 'watching_testrunner', + 'pytest', + # acceptance tests + 'selenium', + ] }, )