Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Try to fix wrong set of LD_LIBRARY_PATH #640

Closed
wants to merge 12 commits into from
3 changes: 3 additions & 0 deletions .github/scripts/build_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@
| | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl |
| | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl |
| | aarch64_musl_dyn | d | | B | B | | |
| | x86-64_musl_static | | | BP | BP | | linux-x86-64-musl |
| | x86-64_musl_mixed | BP | | | | | linux-x86-64-musl |
| | x86-64_musl_dyn | d | | B | B | | |
| | win32_static | d | dB | dBP | dBP | | win-i686 |
| | win32_dyn | d | dB | dB | dB | | |
| | i586_static | | | BP | BP | | linux-i586 |
Expand Down
16 changes: 4 additions & 12 deletions .github/scripts/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
SOURCE_DIR = HOME / "SOURCE"
ARCHIVE_DIR = HOME / "ARCHIVE"
TOOLCHAIN_DIR = HOME / "TOOLCHAINS"
INSTALL_DIR = BASE_DIR / "INSTALL"
TMP_DIR = Path("/tmp")
KBUILD_SOURCE_DIR = HOME / "kiwix-build"
Expand Down Expand Up @@ -278,13 +279,9 @@ def make_deps_archive(target=None, name=None, full=False):
base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt")
# Add ndk/sdk/toolchains to allow project's CI to find them and compile
files_to_archive += HOME.glob("BUILD_*/android-ndk*")
files_to_archive += HOME.glob("BUILD_*/emsdk*")
if PLATFORM_TARGET.startswith("aarch64"):
files_to_archive += SOURCE_DIR.glob("aarch64*/*")
if PLATFORM_TARGET.startswith("armv"):
files_to_archive += SOURCE_DIR.glob("armv*/*")
# Copy any toolchain
files_to_archive += [TOOLCHAIN_DIR]
files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*")
if (BASE_DIR / "meson_cross_file.txt").exists():
files_to_archive.append(BASE_DIR / "meson_cross_file.txt")

Expand All @@ -307,11 +304,6 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive += SOURCE_DIR.glob("*/.*_ok")
files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")

toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
for subdir in toolchains_subdirs:
if not subdir.match("tools"):
files_to_archive.append(subdir)

archive_file = TMP_DIR / archive_name
with tarfile.open(str(archive_file), "w:xz") as tar:
for name in set(files_to_archive):
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ jobs:
- aarch64_musl_static
- aarch64_musl_dyn
- aarch64_musl_mixed
- x86-64_musl_static
- x86-64_musl_dyn
- x86-64_musl_mixed
- i586_static
- i586_dyn
- android_arm
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/releaseNigthly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ jobs:
- aarch64_mixed
- aarch64_musl_static
- aarch64_musl_mixed
- x86-64_musl_static
- x86-64_musl_mixed
- win32_static
- i586_static
- android_arm
Expand Down
9 changes: 4 additions & 5 deletions kiwixbuild/buildenv.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,6 @@ def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):

env['PATH'] = ':'.join([pj(self.install_dir, 'bin'), env['PATH']])

env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])

env['QMAKE_CXXFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['QMAKE_CXXFLAGS']])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
env['QMAKE_LFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
Expand All @@ -142,6 +137,10 @@ def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env['LDFLAGS']])

if cross_comp_flags:
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
self.platformInfo.set_comp_flags(env)
if cross_compilers:
self.platformInfo.set_compiler(env)
Expand Down
10 changes: 5 additions & 5 deletions kiwixbuild/dependencies/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
from .base import *
from . import (
all_dependencies,
android_ndk,
tc_android_ndk,
aria2,
armhf,
arm_musl,
tc_armhf,
tc_musl,
docoptcpp,
emsdk,
flatpak,
tc_emsdk,
tc_flatpak,
gumbo,
icu4c,
ios_fat_lib,
Expand Down
14 changes: 0 additions & 14 deletions kiwixbuild/dependencies/arm_musl.py

This file was deleted.

26 changes: 24 additions & 2 deletions kiwixbuild/dependencies/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import shutil
import time

from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize
from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize, copy_tree
from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option, get_target_step

Expand Down Expand Up @@ -323,6 +323,28 @@ def make_dist(self):
pass


class TcCopyBuilder(Builder):
src_subdir = None

@property
def build_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())

def build(self):
self.command('copy', self._copy)

def _copy(self, context):
context.try_skip(self.build_path)
if self.src_subdir:
source_path = pj(self.source_path, self.src_subdir)
else:
source_path = self.source_path
copy_tree(source_path, self.build_path)

def make_dist(self):
pass


class MakeBuilder(Builder):
configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}"
configure_option = ""
Expand Down Expand Up @@ -541,7 +563,7 @@ def _test(self, context):
):
raise SkipCommand()
command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option)
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
run_command(command, self.build_path, context, env=env)

def _install(self, context):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def source_dir(self):
class Builder(Builder):
@property
def install_path(self):
return self.build_path
return pj(self.buildEnv.toolchain_dir, self.target.full_name())

@property
def api(self):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .base import Dependency, ReleaseDownload, NoopBuilder
from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile

# The arm toolchains
Expand All @@ -19,7 +19,9 @@ class Source(ReleaseDownload):
'4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434',
base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz')

Builder = NoopBuilder

class Builder(TcCopyBuilder):
src_subdir = "armv6-rpi-linux-gnueabihf"


class armv8_toolchain(Dependency):
Expand All @@ -32,7 +34,8 @@ class Source(ReleaseDownload):
'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5',
base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz')

Builder = NoopBuilder
class Builder(TcCopyBuilder):
src_subdir = "armv8-rpi3-linux-gnueabihf"

class aarch64_toolchain(Dependency):
dont_skip = True
Expand All @@ -44,4 +47,4 @@ class Source(ReleaseDownload):
'1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320',
aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz')

Builder = NoopBuilder
Builder = TcCopyBuilder
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,21 @@ def source_dir(self):
class Builder(Builder):
@property
def install_path(self):
return self.build_path
return pj(self.buildEnv.toolchain_dir, self.target.full_name())

def _copy_source(self, context):
context.try_skip(self.build_path)
copy_tree(self.source_path, self.build_path)
copy_tree(self.source_path, self.install_path)

def _install(self, context):
context.try_skip(self.build_path)
command = "./emsdk install 3.1.24"
run_command(command, self.build_path, context)
run_command(command, self.install_path, context)

def _activate(self, context):
context.try_skip(self.build_path)
command = "./emsdk activate 3.1.24"
run_command(command, self.build_path, context)
run_command(command, self.install_path, context)


def build(self):
Expand Down
27 changes: 27 additions & 0 deletions kiwixbuild/dependencies/tc_musl.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile

class aarch64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64_musl"

class Source(ReleaseDownload):
archive = Remotefile('aarch64-linux-musl-cross.tgz',
'0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6',
'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz')

Builder = TcCopyBuilder


class x86_64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "x86-64_musl"

class Source(ReleaseDownload):
archive = Remotefile('x86_64-linux-musl-cross.tgz',
'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8',
'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz')

Builder = TcCopyBuilder
2 changes: 1 addition & 1 deletion kiwixbuild/platforms/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from . import (
android,
armhf,
arm_musl,
musl,
flatpak,
i586,
ios,
Expand Down
8 changes: 4 additions & 4 deletions kiwixbuild/platforms/armhf.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ def libdir(self):
return "lib/{}".format(self.arch_full)

@property
def tlc_source(self):
return get_target_step(self.build, 'source')
def toolchain(self):
return get_target_step(self.build, 'neutral')

@property
def root_path(self):
return pj(self.tlc_source.source_path, self.arch_full)
return self.toolchain.build_path

@property
def binaries(self):
Expand Down Expand Up @@ -150,7 +150,7 @@ class Aarch64(ArmPlatformInfo):

@property
def root_path(self):
return self.tlc_source.source_path
return self.toolchain.build_path

class Aarch64Dyn(Aarch64):
name = 'aarch64_dyn'
Expand Down
Loading
Loading