From 3d40dbbaad64cf7cecc7db1ba83887694029cee4 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Sun, 18 Aug 2024 15:24:42 +0200 Subject: [PATCH 1/8] Install QT on Windows CI --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 069a2105..755c66f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,12 @@ jobs: run: | pip3 install meson pytest requests distro paramiko pip3 install --no-deps $GITHUB_WORKSPACE + - name: Install QT + uses: jurplel/install-qt-action@v4 + with: + version: 5.15.2 + modules: "qtwebengine" + setup-python: false - name: Setup MSVC compiler uses: bus1/cabuild/action/msdevshell@v1 with: From 702292c14ce5ccfc61b93903474e449ae5469a72 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 21 Aug 2024 16:44:53 +0200 Subject: [PATCH 2/8] [CI] Build kiwix-desktop on Windows --- .github/scripts/build_definition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index b9b55f59..f2d5135d 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -24,7 +24,7 @@ # On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll | windows | native_static | Bd | d | | d | | win-x86_64 | win-x86_64-static | | windows | native_dyn | Bd | | | | | win-x86_64 | win-x86_64-dyn | - | windows | native_mixed | BPd | d | | | d | win-x86_64 | win-x86_64-mixed | + | windows | native_mixed | BPd | d | | | Bd | win-x86_64 | win-x86_64-mixed | ---------------------------------------------------------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries | macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn | From 7fbeb2fc6df4c75905381e9b92e8680e4787e40b Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 21 Aug 2024 17:15:12 +0200 Subject: [PATCH 3/8] Move `-j4` option in `make_options` This way we can remove it when on Windows. --- kiwixbuild/dependencies/base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 06e56e6a..de9c98e4 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -2,6 +2,7 @@ import os import shutil import time +import platform from kiwixbuild.utils import ( pj, @@ -379,7 +380,7 @@ class MakeBuilder(Builder): configure_options = [] dynamic_configure_options = ["--enable-shared", "--disable-static"] static_configure_options = ["--enable-static", "--disable-shared"] - make_options = [] + make_options = ["-j4"] install_options = [] configure_script = "configure" configure_env = { @@ -435,7 +436,6 @@ def _compile(self, context): command = [ *self.buildEnv.make_wrapper, *neutralEnv("make_command"), - "-j4", *self.make_targets, *self.make_options, ] @@ -492,6 +492,12 @@ class QMakeBuilder(MakeBuilder): qmake_targets = [] flatpak_buildsystem = "qmake" + @property + def make_options(self): + if platform.system() == "Windows": + return [] + return super().make_options + @property def env_options(self): if "QMAKE_CC" in os.environ: From 1e4b88d2f0f9d74343cc7e9cabadbb3d15395850 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 21 Aug 2024 17:16:29 +0200 Subject: [PATCH 4/8] Use nmake instead of make on Windows Qmake in generating Makefile for nmake on Windows/msvc. --- kiwixbuild/buildenv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index cdd65b98..2ae6ac2e 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -30,7 +30,10 @@ def __init__(self, dummy_run): self.mesontest_command = [*self.meson_command, "test"] self.patch_command = self._detect_command("patch") self.git_command = self._detect_command("git") - self.make_command = self._detect_command("make") + if platform.system() == "Windows": + self.make_command = self._detect_command("nmake", options=["/?", "/NOLOGO"]) + else: + self.make_command = self._detect_command("make") self.cmake_command = self._detect_command("cmake") self.qmake_command = self._detect_command( "qmake", required=False, default=[["qmake"], ["qmake-qt5"]] From ce4d03989b4dd7c2685023fbe76bc542498d477e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 21 Aug 2024 17:23:47 +0200 Subject: [PATCH 5/8] Build only debug or release on Windows. On Windows we cannot mix debug and release build. So we have to choose only one. --- kiwixbuild/dependencies/kiwix_desktop.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py index 67ca3f02..702fdbd0 100644 --- a/kiwixbuild/dependencies/kiwix_desktop.py +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -1,4 +1,6 @@ +from kiwixbuild._global import option from .base import Dependency, GitClone, QMakeBuilder +import platform class KiwixDesktop(Dependency): @@ -11,11 +13,24 @@ class Source(GitClone): class Builder(QMakeBuilder): dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"] - make_install_targets = ["install"] configure_env = None flatpack_build_options = {"env": {"QMAKEPATH": "/app/lib"}} + @property + def make_targets(self): + if platform.system() == "Windows": + yield "release-all" if option("make_release") else "debug-all" + else: + yield from super().make_targets + + @property + def make_install_targets(self): + if platform.system() == "Windows": + yield "release-install" if option("make_release") else "debug-install" + else: + yield "install" + @property def configure_options(self): if self.buildEnv.configInfo.name != "flatpak": From 1d93c35c50aef548afb4abc6efcfbe76671cc5fc Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 23 Aug 2024 10:18:32 +0200 Subject: [PATCH 6/8] Format common.py file. --- .github/scripts/common.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 4bdc1d06..7a2beac2 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -42,7 +42,7 @@ def get_build_dir(config) -> Path: ARCHIVE_DIR = HOME / "ARCHIVE" TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" INSTALL_DIR = BASE_DIR / "INSTALL" -default_tmp_dir = os.getenv("TEMP") if platform.system() == 'Windows' else "/tmp" +default_tmp_dir = os.getenv("TEMP") if platform.system() == "Windows" else "/tmp" TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir)) KBUILD_SOURCE_DIR = HOME / "kiwix-build" @@ -204,6 +204,7 @@ def run_kiwix_build( subprocess.check_call(command, cwd=str(HOME), env=env) print_message("Build ended") + try: import paramiko @@ -216,8 +217,8 @@ def upload(file_to_upload, host, dest_path): host, port = host.split(":", 1) else: port = "22" - if '@' in host: - user, host = host.split('@', 1) + if "@" in host: + user, host = host.split("@", 1) else: user = None @@ -228,7 +229,14 @@ def get_client(): client = paramiko.client.SSHClient() client.set_missing_host_key_policy(paramiko.client.WarningPolicy) print_message(f"Connect to {host}:{port}") - client.connect(host, port=port, username=user, key_filename=_environ.get("SSH_KEY"), look_for_keys=False, compress=True) + client.connect( + host, + port=port, + username=user, + key_filename=_environ.get("SSH_KEY"), + look_for_keys=False, + compress=True, + ) try: yield client finally: @@ -257,7 +265,6 @@ def get_sftp(): print_message(f"Sending archive {file_to_upload} to {remote_file}") sftp.put(str(file_to_upload), str(remote_file), confirm=True) - except ModuleNotFoundError: # On old system (bionic) paramiko is really complex to install # Keep the old implementaion on sush system. From 45ad41724cef27597372fcc3eac826b8c82d68b0 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 22 Aug 2024 16:25:01 +0200 Subject: [PATCH 7/8] Package kiwix-desktop on Windows --- .github/scripts/common.py | 24 ++++++++- .github/workflows/ci.yml | 4 +- scripts/package_kiwix-desktop_windows.py | 63 ++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 scripts/package_kiwix-desktop_windows.py diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 7a2beac2..a3863550 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -44,7 +44,11 @@ def get_build_dir(config) -> Path: INSTALL_DIR = BASE_DIR / "INSTALL" default_tmp_dir = os.getenv("TEMP") if platform.system() == "Windows" else "/tmp" TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir)) -KBUILD_SOURCE_DIR = HOME / "kiwix-build" +if platform.system() == "Windows": + KBUILD_SOURCE_DIR = Path(_environ["GITHUB_WORKSPACE"]) +else: + KBUILD_SOURCE_DIR = HOME / "kiwix-build" + _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None @@ -480,11 +484,27 @@ def create_desktop_image(make_release): build_path = BASE_DIR / "org.kiwix.desktop.flatpak" app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) print_message("archive is {}", build_path) + elif platform.system() == "Windows": + archive_basename = "Kiwix-{}-win-amd64".format(postfix) + working_dir = INSTALL_DIR / archive_basename + build_path = working_dir.with_suffix(".zip") + app_name = build_path.name + command = [ + "python", + KBUILD_SOURCE_DIR / "scripts" / "package_kiwix-desktop_windows.py", + str(INSTALL_DIR), + str(working_dir), + str(build_path), + ] + if make_release: + command += ["-s"] + print_message("Package archive of kiwix-desktop") + subprocess.check_call(command, cwd=str(HOME)) else: build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix) app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) command = [ - "kiwix-build/scripts/create_kiwix-desktop_appImage.sh", + KBUILD_SOURCE_DIR / "scripts" / "create_kiwix-desktop_appImage.sh", str(INSTALL_DIR), str(src_dir), str(HOME / "AppDir"), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 755c66f0..86f0c141 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,10 +22,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - - name: Setup python 3.8 + - name: Setup python 3.12 uses: actions/setup-python@v3 with: - python-version: '3.8' + python-version: '3.12' - name: Install packages run: | choco.exe install pkgconfiglite ninja diff --git a/scripts/package_kiwix-desktop_windows.py b/scripts/package_kiwix-desktop_windows.py new file mode 100644 index 00000000..0c4ceb8d --- /dev/null +++ b/scripts/package_kiwix-desktop_windows.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import sys, subprocess, shutil, argparse +from pathlib import Path + +parser = argparse.ArgumentParser() + +parser.add_argument("install_dir") +parser.add_argument("out_dir") +parser.add_argument("--static_dir") +parser.add_argument("archive_path", help="The full path of the archive to create") +parser.add_argument("-s", "--sign", action="store_true") + +args = parser.parse_args() + +install_dir = Path(args.install_dir) +if args.static_dir: + static_dir = Path(args.static_dir) +else: + static_dir = install_dir + +out_dir = Path(args.out_dir) +archive_path = Path(args.archive_path) + +out_dir.mkdir(parents=True, exist_ok=True) + +print( + f"""Packaging kiwix-desktop +- From {install_dir} +- Static dir is {static_dir} +- Working dir {out_dir} +- Archive path is {archive_path} +- Python version {sys.version}""" +) + +shutil.copy2(install_dir / "bin" / "kiwix-desktop.exe", out_dir) +subprocess.run(["windeployqt", "--compiler-runtime", str(out_dir)], check=True) + + +shutil.copy2(static_dir / "bin" / "aria2c.exe", out_dir) + +for dll in (static_dir / "bin").glob("*.dll"): + shutil.copy2(dll, out_dir) + + +# Copy ssl stuff +ssl_directory = Path("C:/") / "Program Files" / "OpenSSL" +shutil.copy2(ssl_directory / "libcrypto-1_1-x64.dll", out_dir) +shutil.copy2(ssl_directory / "libssl-1_1-x64.dll", out_dir) + +# [TODO] Sign binary +if args.sign: + pass + +print( + f"""Create archive +- {archive_path.with_suffix('')} +- From {out_dir.parent} +- With {out_dir.name}""" +) +shutil.make_archive( + archive_path.with_suffix(""), "zip", root_dir=out_dir.parent, base_dir=out_dir.name +) From d8b4d24d7f86f5dc46c363924b6a4deed8327111 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 23 Aug 2024 18:07:50 +0200 Subject: [PATCH 8/8] Add more dependencies is base dependencies on Windows --- kiwixbuild/dependencies/all_dependencies.py | 68 +++++++++++++-------- kiwixbuild/versions.py | 2 +- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 25dba4fc..3a364c70 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -13,32 +13,50 @@ class AllBaseDependencies(Dependency): class Builder(NoopBuilder): @classmethod def get_dependencies(cls, configInfo, allDeps): - if neutralEnv("distname") == "Windows": - return ["zlib", "zstd", "icu4c", "zim-testing-suite", "xapian-core"] if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux": return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"] - base_deps = [ - "zlib", - "lzma", - "zstd", - "xapian-core", - "pugixml", - "libcurl", - "icu4c", - "mustache", - "libmicrohttpd", - "zim-testing-suite", - ] - # Add specific dependencies depending of the config - if configInfo.build not in ("android", "iOS"): - # For zimtools - base_deps += ["docoptcpp"] - if configInfo.build != "win32": - # zimwriterfs - base_deps += ["libmagic", "gumbo"] - if configInfo.build == "native" and neutralEnv("distname") != "Darwin": - # We compile kiwix-desktop only on native and not on `Darwin` - # So we need aria2 only there - base_deps += ["aria2"] + if neutralEnv("distname") == "Windows": + base_deps = [ + "zlib", + "zstd", + "xapian-core", + "zim-testing-suite", + "icu4c", + ] + + if not configInfo.name.endswith("_dyn"): + base_deps += [ + "pugixml", + "libcurl", + "mustache", + "libmicrohttpd", + ] + else: + base_deps = [ + "zlib", + "lzma", + "zstd", + "xapian-core", + "pugixml", + "libcurl", + "icu4c", + "mustache", + "libmicrohttpd", + "zim-testing-suite", + ] + # Add specific dependencies depending of the config + if configInfo.build not in ("android", "iOS"): + # For zimtools + base_deps += ["docoptcpp"] + if configInfo.build != "win32": + # zimwriterfs + base_deps += ["libmagic", "gumbo"] + if ( + configInfo.build == "native" + and neutralEnv("distname") != "Darwin" + ): + # We compile kiwix-desktop only on native and not on `Darwin` + # So we need aria2 only there + base_deps += ["aria2"] return base_deps diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index feabcc18..e1ad6956 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = "04" +base_deps_meta_version = "05" base_deps_versions = { "zlib": "1.2.12",