From 6f1a53accea0f0a357f80cbf5a3fc80a29ca2d1f Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 2 Oct 2023 09:49:40 +0200 Subject: [PATCH 1/3] Split Python dependencies installation in Docker build --- CHANGELOG.md | 3 +++ Dockerfile | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 773aa1e..32b9c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Dockerfile: split installation of Python dependencies for more efficiency + ## [1.1.0] - 2023-07-25 ### Added diff --git a/Dockerfile b/Dockerfile index 743bd2b..2b96101 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,16 @@ RUN apt-get update \ && python -m pip install --no-cache-dir -U \ pip +# Copy pyproject.toml and its dependencies +COPY pyproject.toml README.md get_js_deps.sh hatch_build.py /src/ +COPY src/kolibri2zim/__about__.py /src/src/kolibri2zim/__about__.py + +# Install Python dependencies +RUN pip install --no-cache-dir /src + # Copy code + associated artifacts COPY src /src/src -COPY pyproject.toml *.md get_js_deps.sh MANIFEST.in LICENSE *.py /src/ +COPY *.md LICENSE *.py /src/ # Install + cleanup RUN pip install --no-cache-dir /src \ From a0bdcdf773abb1fe93ea041d1a470ecbcab9adcc Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 2 Oct 2023 09:53:31 +0200 Subject: [PATCH 2/3] Add html option in coverage report + small fix in invoke tasks --- CHANGELOG.md | 6 ++++ pyproject.toml | 1 + tasks.py | 79 +++++++++++++++++++++++++++----------------------- 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32b9c1e..6b40338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- New html option in coverage report + +### Fixed +- Small fixes in invoke tasks + ### Changed - Dockerfile: split installation of Python dependencies for more efficiency diff --git a/pyproject.toml b/pyproject.toml index 944c679..a002b7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ run = "inv test --args '{args}'" run-cov = "inv test-cov --args '{args}'" report-cov = "inv report-cov" coverage = "inv coverage --args '{args}'" +html = "inv coverage --html --args '{args}'" [tool.hatch.envs.lint] template = "lint" diff --git a/tasks.py b/tasks.py index 3370b73..90854e8 100644 --- a/tasks.py +++ b/tasks.py @@ -8,95 +8,102 @@ @task(optional=["args"], help={"args": "pytest additional arguments"}) -def test(ctx: Context, args: str | None = ""): +def test(ctx: Context, args: str = ""): """run tests (without coverage)""" ctx.run(f"pytest {args}", pty=use_pty) @task(optional=["args"], help={"args": "pytest additional arguments"}) -def test_cov(ctx: Context, args: str | None = ""): +def test_cov(ctx: Context, args: str = ""): """run test vith coverage""" ctx.run(f"coverage run -m pytest {args}", pty=use_pty) -@task() -def report_cov(ctx: Context): +@task(optional=["html"], help={"html": "flag to export html report"}) +def report_cov(ctx: Context, *, html: bool = False): """report coverage""" ctx.run("coverage combine", warn=True, pty=use_pty) ctx.run("coverage report --show-missing", pty=use_pty) + if html: + ctx.run("coverage html", pty=use_pty) -@task(optional=["args"], help={"args": "pytest additional arguments"}) -def coverage(ctx: Context, args: str | None = ""): +@task( + optional=["args", "html"], + help={ + "args": "pytest additional arguments", + "html": "flag to export html report", + }, +) +def coverage(ctx: Context, args: str = "", *, html: bool = False): """run tests and report coverage""" - test_cov(ctx, args) - report_cov(ctx) + test_cov(ctx, args=args) + report_cov(ctx, html=html) -@task( - optional=["args"], help={"args": "linting tools (black, ruff) additional arguments"} -) -def lint_black(ctx: Context, args: str | None = ""): - args = args or "." +@task(optional=["args"], help={"args": "black additional arguments"}) +def lint_black(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script ctx.run("black --version", pty=use_pty) ctx.run(f"black --check --diff {args}", pty=use_pty) -@task( - optional=["args"], help={"args": "linting tools (black, ruff) additional arguments"} -) -def lint_ruff(ctx: Context, args: str | None = ""): - args = args or "." +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def lint_ruff(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script ctx.run("ruff --version", pty=use_pty) ctx.run(f"ruff check {args}", pty=use_pty) @task( - optional=["args"], help={"args": "linting tools (black, ruff) additional arguments"} + optional=["args"], + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, ) -def lintall(ctx: Context, args: str | None = ""): - """check linting""" - args = args or "." +def lintall(ctx: Context, args: str = "."): + """Check linting""" + args = args or "." # needed for hatch script lint_black(ctx, args) lint_ruff(ctx, args) @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def check_pyright(ctx: Context, args: str | None = ""): +def check_pyright(ctx: Context, args: str = ""): """check static types with pyright""" - args = args or "" ctx.run("pyright --version") ctx.run(f"pyright {args}", pty=use_pty) @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def checkall(ctx: Context, args: str | None = ""): +def checkall(ctx: Context, args: str = ""): """check static types""" - args = args or "" check_pyright(ctx, args) @task(optional=["args"], help={"args": "black additional arguments"}) -def fix_black(ctx: Context, args: str | None = ""): +def fix_black(ctx: Context, args: str = "."): """fix black formatting""" - args = args or "." - ctx.run(f"black {args}", pty=use_pty) # type: ignore + args = args or "." # needed for hatch script + ctx.run(f"black {args}", pty=use_pty) @task(optional=["args"], help={"args": "ruff additional arguments"}) -def fix_ruff(ctx: Context, args: str | None = ""): +def fix_ruff(ctx: Context, args: str = "."): """fix all ruff rules""" - args = args or "." - ctx.run(f"ruff --fix {args}", pty=use_pty) # type: ignore + args = args or "." # needed for hatch script + ctx.run(f"ruff --fix {args}", pty=use_pty) @task( optional=["args"], - help={"args": "linting (fix mode) tools (black, ruff) additional arguments"}, + help={ + "args": "linting tools (black, ruff) additional arguments, typically a path", + }, ) -def fixall(ctx: Context, args: str | None = ""): - """fix everything automatically""" - args = args or "." +def fixall(ctx: Context, args: str = "."): + """Fix everything automatically""" + args = args or "." # needed for hatch script fix_black(ctx, args) fix_ruff(ctx, args) lintall(ctx, args) From 9cf26b9dfa2ba1838aaa22d4a892b9e25290989e Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 2 Oct 2023 09:53:55 +0200 Subject: [PATCH 3/3] Adopt Python bootstrap standards for CI --- .../workflows/{publish.yml => Publish.yml} | 2 +- .github/workflows/PublishDockerDevImage.yaml | 27 +++++++++++++++++ .github/workflows/{qa.yml => QA.yml} | 5 +++- .github/workflows/Tests.yml | 30 +++++++++++++++++-- .github/workflows/pull.yml | 10 ------- .github/workflows/push.yml | 8 ----- CHANGELOG.md | 3 ++ 7 files changed, 62 insertions(+), 23 deletions(-) rename .github/workflows/{publish.yml => Publish.yml} (94%) create mode 100644 .github/workflows/PublishDockerDevImage.yaml rename .github/workflows/{qa.yml => QA.yml} (91%) delete mode 100644 .github/workflows/pull.yml delete mode 100644 .github/workflows/push.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/Publish.yml similarity index 94% rename from .github/workflows/publish.yml rename to .github/workflows/Publish.yml index 7c82c0a..7061f40 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/Publish.yml @@ -8,7 +8,7 @@ jobs: publish: runs-on: ubuntu-22.04 permissions: - id-token: write # mandatory for PyPI trusted publishing + id-token: write # mandatory for PyPI trusted publishing steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/PublishDockerDevImage.yaml b/.github/workflows/PublishDockerDevImage.yaml new file mode 100644 index 0000000..e363cf4 --- /dev/null +++ b/.github/workflows/PublishDockerDevImage.yaml @@ -0,0 +1,27 @@ +name: Publish Docker dev image + +on: + push: + branches: + - main + +jobs: + publish: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + + - name: Build and push Docker image + uses: openzim/docker-publish-action@v10 + with: + image-name: openzim/kolibri + manual-tag: dev + latest-on-tag: false + restrict-to: openzim/kolibri + registries: ghcr.io + credentials: + GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} + repo_description: auto + repo_overview: auto diff --git a/.github/workflows/qa.yml b/.github/workflows/QA.yml similarity index 91% rename from .github/workflows/qa.yml rename to .github/workflows/QA.yml index 54c93eb..a9d2172 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/QA.yml @@ -1,7 +1,10 @@ name: QA on: - workflow_call: + pull_request: + push: + branches: + - main jobs: check-qa: diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 53ace14..838269f 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -32,7 +32,31 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} - - name: Ensure we can build targets + build_python: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Ensure we can build Python targets + run: | + pip install -U pip build + python3 -m build --sdist --wheel + + build_docker: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Ensure we can build the Docker image + run: | + docker build -t testimage . + + - name: Ensure we can start the Docker image run: | - pip install build - python3 -m build + docker run --rm testimage diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml deleted file mode 100644 index be2b9de..0000000 --- a/.github/workflows/pull.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Pull Request - -on: - pull_request: - -jobs: - qa: - uses: ./.github/workflows/qa.yml - # run qa job if the pull request originates from a fork (otherwise the qa is already triggered by the push to a branch) - if: github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml deleted file mode 100644 index d84f392..0000000 --- a/.github/workflows/push.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Push - -on: - push: - -jobs: - qa: - uses: ./.github/workflows/qa.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b40338..59efa03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Dockerfile: split installation of Python dependencies for more efficiency +- Github workflow: publish `dev` tag on every push to `main` branch +- Github workflow: build Docker image + test its startup +- Github workflow: adopt new standard execution structure (`on` conditions) ## [1.1.0] - 2023-07-25