diff --git a/ci_test.py b/ci_test.py index a264fb1d..ee4da17f 100755 --- a/ci_test.py +++ b/ci_test.py @@ -16,19 +16,22 @@ import urllib.request import json import subprocess +import shlex import sys import os def run_command(cmd, log_file=None): - print("Running: {}".format(cmd)) + if isinstance(cmd, str): + cmd = shlex.split(cmd) + print("Running: {}".format(shlex.join(cmd))) sys.stdout.flush() if log_file: file = open(log_file, "w") - p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file) else: - p = subprocess.Popen(cmd, shell=True) - + file = None + p = subprocess.Popen(cmd, stdout=file, stderr=file) + (output, err) = p.communicate() return p.wait() @@ -62,22 +65,37 @@ def main(): results = {} suite_status = True dockerfiles = get_dockerfiles() + root_dir = os.path.dirname(os.path.realpath(__file__)) for dockerfile in dockerfiles: - docker_dir = os.path.dirname(os.path.realpath(__file__)) + # Make sure everything is relative + dockerfile = os.path.relpath(os.path.realpath(dockerfile), root_dir) + + docker_dir, docker_name = os.path.split(dockerfile) + print("Testing {}".format(dockerfile)) sys.stdout.flush() - log_file = dockerfile.replace(docker_dir,"").replace("/", "_") + log_file = dockerfile.replace("/", "_") log_file = "{}.log".format(log_file) - cmd = "docker build --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile) + cmd = [ + 'docker', 'build', '--no-cache=true', + '-f', dockerfile, + docker_dir + ] if "buildx" in dockerfile: # if "buildx" is part of the path, we want to use the new buildx build system and build # for both amd64 and arm64. - cmd = "docker buildx create --use" - run_command(cmd, log_file) - cmd = "docker buildx inspect --bootstrap" - run_command(cmd, log_file) - cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile) - status = run_command(cmd, log_file) + run_command("docker buildx create --use", log_file=log_file) + run_command("docker buildx inspect --bootstrap", log_file=log_file) + + cmd = [ + 'docker', 'buildx', 'build', + '--platform', 'linux/arm64,linux/amd64', + '--no-cache=true', + '-f', dockerfile, + docker_dir + ] + + status = run_command(cmd, log_file=log_file) results[dockerfile] = status if status != 0: suite_status = False @@ -85,7 +103,9 @@ def main(): else: results[dockerfile] = "PASSED" - cmd = "mv {log} {results}{log}".format(log=log_file, results=results[dockerfile]) + cmd = [ + 'mv', log_file, results[dockerfile] + log_file + ] run_command(cmd) print("[{}] - {}".format(results[dockerfile], dockerfile)) sys.stdout.flush() diff --git a/swift-ci/sdks/static-linux/Dockerfile b/swift-ci/sdks/static-linux/Dockerfile index e16e8c42..5253602b 100644 --- a/swift-ci/sdks/static-linux/Dockerfile +++ b/swift-ci/sdks/static-linux/Dockerfile @@ -24,6 +24,9 @@ ARG BORINGSSL_VERSION=fips-20220613 ARG ICU_VERSION=maint/maint-69 ARG ZLIB_VERSION=1.3.1 +# Architecture to build on (empty means x86-64) +ARG OS_ARCH_SUFFIX= + # ............................................................................ # Install development tools @@ -62,7 +65,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ OS_MAJOR_VER=$OS_MAJOR_VER \ OS_MINOR_VER=$OS_MINOR_VER \ OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER \ - SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER" + SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER$OS_ARCH_SUFFIX" COPY scripts/install-swift.sh /scripts/install-swift.sh RUN chmod ugo+x /scripts/install-swift.sh diff --git a/swift-ci/sdks/static-linux/build b/swift-ci/sdks/static-linux/build index 7bbc2bf0..6e984960 100755 --- a/swift-ci/sdks/static-linux/build +++ b/swift-ci/sdks/static-linux/build @@ -14,10 +14,25 @@ # # ===----------------------------------------------------------------------=== -DOCKER=docker +if [[ "$DOCKER" == "" ]]; then + DOCKER=docker +fi + +case $(arch) in + arm64|aarch64) + OS_ARCH_SUFFIX=-aarch64 + ;; + amd64|x86_64) + OS_ARCH_SUFFIX= + ;; + *) + echo "Unknown architecture $(arch)" + exit 1 + ;; +esac # Build the Docker image -$(DOCKER) build -t static-swift-linux . +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX -t static-swift-linux . # Check-out the sources scripts/fetch-source.sh --clone-with-ssh --source-dir source @@ -25,7 +40,7 @@ scripts/fetch-source.sh --clone-with-ssh --source-dir source mkdir -p products # Run the build -$(DOCKER) run -it --rm \ +$DOCKER run -it --rm \ -v ./source:/source \ -v ./products:/products \ static-swift-linux \ diff --git a/swift-ci/sdks/static-linux/scripts/build.sh b/swift-ci/sdks/static-linux/scripts/build.sh index 798d5381..4012b6ac 100755 --- a/swift-ci/sdks/static-linux/scripts/build.sh +++ b/swift-ci/sdks/static-linux/scripts/build.sh @@ -144,6 +144,20 @@ while [ "$#" -gt 0 ]; do shift done +# Work out the host architecture +case $(arch) in + arm64|aarch64) + host_arch=arm64 + ;; + amd64|x86_64) + host_arch=x86_64 + ;; + *) + echo "Unknown host architecture $(arch)" + exit 1 + ;; +esac + # Change the commas for spaces archs="${archs//,/ }" @@ -662,7 +676,7 @@ EOF --compiler-vendor=apple \ --bootstrapping hosttools \ --build-linux-static --install-swift \ - --stdlib-deployment-targets linux-x86_64,linux-static-$arch \ + --stdlib-deployment-targets linux-$host_arch,linux-static-$arch \ --build-stdlib-deployment-targets all \ --musl-path=${build_dir}/sdk_root \ --linux-static-arch=$arch \ @@ -724,6 +738,7 @@ EOF -DFOUNDATION_PATH_TO_LIBDISPATCH_BUILD=${build_dir}/$arch/dispatch \ -D_SwiftFoundation_SourceDIR=${source_dir}/swift-project/swift-foundation \ -D_SwiftFoundationICU_SourceDIR=${source_dir}/swift-project/swift-foundation-icu \ + -D_SwiftCollections_SourceDIR=${source_dir}/swift-project/swift-collections \ -DCMAKE_Swift_COMPILER_WORKS=YES \ -Ddispatch_DIR=${build_dir}/$arch/dispatch/cmake/modules diff --git a/swift-ci/sdks/static-linux/scripts/fetch-source.sh b/swift-ci/sdks/static-linux/scripts/fetch-source.sh index edda203d..7eb5827a 100755 --- a/swift-ci/sdks/static-linux/scripts/fetch-source.sh +++ b/swift-ci/sdks/static-linux/scripts/fetch-source.sh @@ -164,7 +164,7 @@ cd swift # Get its dependencies header "Fetching Swift Dependencies" -extra_args=--skip-history +extra_args="--skip-history --all-repositories" if [[ $SWIFT_VERSION == scheme:* ]]; then utils/update-checkout ${clone_arg} --scheme ${SWIFT_VERSION#scheme:} ${extra_args} elif [[ $SWIFT_VERSION == tag:* ]]; then