[Static Linux SDK] Build script improvements.

ci_test.py

@@ -16,19 +16,22 @@
 import urllib.request
 import json
 import subprocess
+import shlex
 import sys
 import os
 
 
 def run_command(cmd, log_file=None):
-    print("Running: {}".format(cmd))
+    if isinstance(cmd, str):
+        cmd = shlex.split(cmd)
+    print("Running: {}".format(shlex.join(cmd)))
     sys.stdout.flush()
     if log_file:
         file = open(log_file, "w")
-        p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file)
     else:
-        p = subprocess.Popen(cmd, shell=True)
-
+        file = None
+    p = subprocess.Popen(cmd, stdout=file, stderr=file)
+
     (output, err) = p.communicate()
     return p.wait()
 
@@ -62,30 +65,47 @@ def main():
     results = {}
     suite_status = True
     dockerfiles = get_dockerfiles()
+    root_dir = os.path.dirname(os.path.realpath(__file__))
     for dockerfile in dockerfiles:
-        docker_dir = os.path.dirname(os.path.realpath(__file__))
+        # Make sure everything is relative
+        dockerfile = os.path.relpath(os.path.realpath(dockerfile), root_dir)
+
+        docker_dir, docker_name = os.path.split(dockerfile)
+
         print("Testing {}".format(dockerfile))
         sys.stdout.flush()
-        log_file = dockerfile.replace(docker_dir,"").replace("/", "_")
+        log_file = dockerfile.replace("/", "_")
         log_file = "{}.log".format(log_file)
-        cmd = "docker build --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile)
+        cmd = [
+            'docker', 'build', '--no-cache=true',
+            '-f', dockerfile,
+            docker_dir
+        ]
         if "buildx" in dockerfile:
             # if "buildx" is part of the path, we want to use the new buildx build system and build
             # for both amd64 and arm64.
-            cmd = "docker buildx create --use"
-            run_command(cmd, log_file)
-            cmd = "docker buildx inspect --bootstrap"
-            run_command(cmd, log_file)
-            cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile)
-        status = run_command(cmd, log_file)
+            run_command("docker buildx create --use", log_file=log_file)
+            run_command("docker buildx inspect --bootstrap", log_file=log_file)
+
+            cmd = [
+                'docker', 'buildx', 'build',
+                '--platform', 'linux/arm64,linux/amd64',
+                '--no-cache=true',
+                '-f', dockerfile,
+                docker_dir
+            ]
+
+        status = run_command(cmd, log_file=log_file)
         results[dockerfile] = status
         if status != 0:
             suite_status = False
             results[dockerfile] = "FAILED"
         else:
             results[dockerfile] = "PASSED"
 
-        cmd = "mv {log} {results}{log}".format(log=log_file, results=results[dockerfile])
+        cmd = [
+            'mv', log_file, results[dockerfile] + log_file
+        ]
         run_command(cmd)
         print("[{}] - {}".format(results[dockerfile], dockerfile))
         sys.stdout.flush()

swift-ci/sdks/static-linux/Dockerfile

@@ -24,6 +24,9 @@ ARG BORINGSSL_VERSION=fips-20220613
 ARG ICU_VERSION=maint/maint-69
 ARG ZLIB_VERSION=1.3.1
 
+# Architecture to build on (empty means x86-64)
+ARG OS_ARCH_SUFFIX=
+
 # ............................................................................
 
 # Install development tools
@@ -62,7 +65,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
     OS_MAJOR_VER=$OS_MAJOR_VER \
     OS_MINOR_VER=$OS_MINOR_VER \
     OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER \
-    SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER"
+    SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER$OS_ARCH_SUFFIX"
 
 COPY scripts/install-swift.sh /scripts/install-swift.sh
 RUN chmod ugo+x /scripts/install-swift.sh

swift-ci/sdks/static-linux/build

@@ -14,18 +14,33 @@
 #
 # ===----------------------------------------------------------------------===
 
-DOCKER=docker
+if [[ "$DOCKER" == "" ]]; then
+    DOCKER=docker
+fi
+
+case $(arch) in
+    arm64|aarch64)
+        OS_ARCH_SUFFIX=-aarch64
+        ;;
+    amd64|x86_64)
+        OS_ARCH_SUFFIX=
+        ;;
+    *)
+        echo "Unknown architecture $(arch)"
+        exit 1
+        ;;
+esac
 
 # Build the Docker image
-$(DOCKER) build -t static-swift-linux .
+$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX -t static-swift-linux .
 
 # Check-out the sources
 scripts/fetch-source.sh --clone-with-ssh --source-dir source
 
 mkdir -p products
 
 # Run the build
-$(DOCKER) run -it --rm \
+$DOCKER run -it --rm \
           -v ./source:/source \
           -v ./products:/products \
           static-swift-linux \

swift-ci/sdks/static-linux/scripts/build.sh

@@ -144,6 +144,20 @@ while [ "$#" -gt 0 ]; do
     shift
 done
 
+# Work out the host architecture
+case $(arch) in
+    arm64|aarch64)
+        host_arch=arm64
+        ;;
+    amd64|x86_64)
+        host_arch=x86_64
+        ;;
+    *)
+        echo "Unknown host architecture $(arch)"
+        exit 1
+        ;;
+esac
+
 # Change the commas for spaces
 archs="${archs//,/ }"
 
@@ -662,7 +676,7 @@ EOF
         --compiler-vendor=apple \
         --bootstrapping hosttools \
         --build-linux-static --install-swift \
-        --stdlib-deployment-targets linux-x86_64,linux-static-$arch \
+        --stdlib-deployment-targets linux-$host_arch,linux-static-$arch \
         --build-stdlib-deployment-targets all \
         --musl-path=${build_dir}/sdk_root \
         --linux-static-arch=$arch \
@@ -724,6 +738,7 @@ EOF
           -DFOUNDATION_PATH_TO_LIBDISPATCH_BUILD=${build_dir}/$arch/dispatch \
           -D_SwiftFoundation_SourceDIR=${source_dir}/swift-project/swift-foundation \
           -D_SwiftFoundationICU_SourceDIR=${source_dir}/swift-project/swift-foundation-icu \
+          -D_SwiftCollections_SourceDIR=${source_dir}/swift-project/swift-collections \
           -DCMAKE_Swift_COMPILER_WORKS=YES \
           -Ddispatch_DIR=${build_dir}/$arch/dispatch/cmake/modules
 

swift-ci/sdks/static-linux/scripts/fetch-source.sh

@@ -164,7 +164,7 @@ cd swift
 # Get its dependencies
 header "Fetching Swift Dependencies"
 
-extra_args=--skip-history
+extra_args="--skip-history --all-repositories"
 if [[ $SWIFT_VERSION == scheme:* ]]; then
     utils/update-checkout ${clone_arg} --scheme ${SWIFT_VERSION#scheme:} ${extra_args}
 elif [[ $SWIFT_VERSION == tag:* ]]; then