From 93f8b6108a1a1e76710295388d98507a8fc80953 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 20 Oct 2023 14:06:16 +0200 Subject: [PATCH] Move all toolchain installation in TOOLCHAIN directory. This is a clearer install and it simplify the base_deps archive creation. --- .github/scripts/common.py | 15 +++----------- kiwixbuild/dependencies/base.py | 24 ++++++++++++++++++++++- kiwixbuild/dependencies/tc_android_ndk.py | 2 +- kiwixbuild/dependencies/tc_armhf.py | 11 +++++++---- kiwixbuild/dependencies/tc_emsdk.py | 2 +- kiwixbuild/dependencies/tc_musl.py | 6 +++--- kiwixbuild/platforms/armhf.py | 8 ++++---- kiwixbuild/platforms/musl.py | 6 +++--- kiwixbuild/versions.py | 2 +- 9 files changed, 46 insertions(+), 30 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 83561f32..c0341e9a 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -27,6 +27,7 @@ BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" +TOOLCHAIN_DIR = HOME / "TOOLCHAINS" INSTALL_DIR = BASE_DIR / "INSTALL" TMP_DIR = Path("/tmp") KBUILD_SOURCE_DIR = HOME / "kiwix-build" @@ -278,13 +279,8 @@ def make_deps_archive(target=None, name=None, full=False): base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") - # Add ndk/sdk/toolchains to allow project's CI to find them and compile - files_to_archive += HOME.glob("BUILD_*/android-ndk*") - files_to_archive += HOME.glob("BUILD_*/emsdk*") - if PLATFORM_TARGET.startswith("aarch64"): - files_to_archive += SOURCE_DIR.glob("aarch64*/*") - if PLATFORM_TARGET.startswith("armv"): - files_to_archive += SOURCE_DIR.glob("armv*/*") + # Copy any toolchain + files_to_archive += [TOOLCHAIN_DIR] if (BASE_DIR / "meson_cross_file.txt").exists(): files_to_archive.append(BASE_DIR / "meson_cross_file.txt") @@ -307,11 +303,6 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += SOURCE_DIR.glob("*/.*_ok") files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*") - toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") - for subdir in toolchains_subdirs: - if not subdir.match("tools"): - files_to_archive.append(subdir) - archive_file = TMP_DIR / archive_name with tarfile.open(str(archive_file), "w:xz") as tar: for name in set(files_to_archive): diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 3b5ae2ff..68b8a81b 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -3,7 +3,7 @@ import shutil import time -from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize +from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize, copy_tree from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option, get_target_step @@ -323,6 +323,28 @@ def make_dist(self): pass +class TcCopyBuilder(Builder): + src_subdir = None + + @property + def build_path(self): + return pj(self.buildEnv.toolchain_dir, self.target.full_name()) + + def build(self): + self.command('copy', self._copy) + + def _copy(self, context): + context.try_skip(self.build_path) + if self.src_subdir: + source_path = pj(self.source_path, self.src_subdir) + else: + source_path = self.source_path + copy_tree(source_path, self.build_path) + + def make_dist(self): + pass + + class MakeBuilder(Builder): configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}" configure_option = "" diff --git a/kiwixbuild/dependencies/tc_android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py index 88cad290..5d695e07 100644 --- a/kiwixbuild/dependencies/tc_android_ndk.py +++ b/kiwixbuild/dependencies/tc_android_ndk.py @@ -25,7 +25,7 @@ def source_dir(self): class Builder(Builder): @property def install_path(self): - return self.build_path + return pj(self.buildEnv.toolchain_dir, self.target_full_name()) @property def api(self): diff --git a/kiwixbuild/dependencies/tc_armhf.py b/kiwixbuild/dependencies/tc_armhf.py index 0080f6d7..507aee17 100644 --- a/kiwixbuild/dependencies/tc_armhf.py +++ b/kiwixbuild/dependencies/tc_armhf.py @@ -1,4 +1,4 @@ -from .base import Dependency, ReleaseDownload, NoopBuilder +from .base import Dependency, ReleaseDownload, TcCopyBuilder from kiwixbuild.utils import Remotefile # The arm toolchains @@ -19,7 +19,9 @@ class Source(ReleaseDownload): '4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434', base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz') - Builder = NoopBuilder + + class Builder(TcCopyBuilder): + src_subdir = "armv6-rpi-linux-gnueabihf" class armv8_toolchain(Dependency): @@ -32,7 +34,8 @@ class Source(ReleaseDownload): 'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5', base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz') - Builder = NoopBuilder + class Builder(TcCopyBuilder): + src_subdir = "armv8-rpi3-linux-gnueabihf" class aarch64_toolchain(Dependency): dont_skip = True @@ -44,4 +47,4 @@ class Source(ReleaseDownload): '1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320', aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz') - Builder = NoopBuilder + Builder = TcCopyBuilder diff --git a/kiwixbuild/dependencies/tc_emsdk.py b/kiwixbuild/dependencies/tc_emsdk.py index bd7c2834..43caaa9a 100644 --- a/kiwixbuild/dependencies/tc_emsdk.py +++ b/kiwixbuild/dependencies/tc_emsdk.py @@ -23,7 +23,7 @@ def source_dir(self): class Builder(Builder): @property def install_path(self): - return self.build_path + return pj(self.buildEnv.toolchain_dir, self.target.full_name()) def _copy_source(self, context): context.try_skip(self.build_path) diff --git a/kiwixbuild/dependencies/tc_musl.py b/kiwixbuild/dependencies/tc_musl.py index cace7fce..4ca68f52 100644 --- a/kiwixbuild/dependencies/tc_musl.py +++ b/kiwixbuild/dependencies/tc_musl.py @@ -1,4 +1,4 @@ -from .base import Dependency, ReleaseDownload, NoopBuilder +from .base import Dependency, ReleaseDownload, TcCopyBuilder from kiwixbuild.utils import Remotefile class aarch64_musl_toolchain(Dependency): @@ -11,7 +11,7 @@ class Source(ReleaseDownload): '0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6', 'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz') - Builder = NoopBuilder + Builder = TcCopyBuilder class x86_64_musl_toolchain(Dependency): @@ -24,4 +24,4 @@ class Source(ReleaseDownload): 'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8', 'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz') - Builder = NoopBuilder + Builder = TcCopyBuilder diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 2d1e0ac3..af980c93 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -30,12 +30,12 @@ def libdir(self): return "lib/{}".format(self.arch_full) @property - def tlc_source(self): - return get_target_step(self.build, 'source') + def toolchain(self): + return get_target_step(self.build, 'build') @property def root_path(self): - return pj(self.tlc_source.source_path, self.arch_full) + return pj(self.toolchain.build_path, self.arch_full) @property def binaries(self): @@ -150,7 +150,7 @@ class Aarch64(ArmPlatformInfo): @property def root_path(self): - return self.tlc_source.source_path + return self.toolchain.build_path class Aarch64Dyn(Aarch64): name = 'aarch64_dyn' diff --git a/kiwixbuild/platforms/musl.py b/kiwixbuild/platforms/musl.py index 25ab23a4..d1e4458b 100644 --- a/kiwixbuild/platforms/musl.py +++ b/kiwixbuild/platforms/musl.py @@ -25,12 +25,12 @@ def get_cross_config(self): } @property - def tlc_source(self): - return get_target_step(self.build, 'source') + def toolchain(self): + return get_target_step(self.build, 'build') @property def root_path(self): - return self.tlc_source.source_path + return self.toolchain.build_path @property def binaries(self): diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index b3db0135..1c7421ca 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '92' +base_deps_meta_version = '92_dev' base_deps_versions = { 'zlib' : '1.2.12',