From 62c8581d0e2adbf2501d9a28981abe4903d4b9eb Mon Sep 17 00:00:00 2001
From: Alastair Houghton <ahoughton@apple.com>
Date: Wed, 18 Sep 2024 15:49:52 +0100
Subject: [PATCH] [Static Linux SDK] Build script improvements. (#417)

* [Static Linux SDK] Build script improvements.

When running on an aarch64 host, we need to download a different version
of the Swift compiler in order to do the build.  If we run the build on
a macOS host, we also have to tell the update-checkout script to fetch
all the required dependencies (even Linux-only ones).

Also tell Foundation where swift-collections lives, to avoid it fetching
another copy.

* [CI] Run Docker builds in the right directory.

Make sure we run the Docker builds in the directory containing the
Dockerfile.  Also, prefer lists of arguments rather than strings,
especially when including filenames.
---
 ci_test.py                                    | 48 +++++++++++++------
 swift-ci/sdks/static-linux/Dockerfile         |  5 +-
 swift-ci/sdks/static-linux/build              | 21 ++++++--
 swift-ci/sdks/static-linux/scripts/build.sh   | 17 ++++++-
 .../sdks/static-linux/scripts/fetch-source.sh |  2 +-
 5 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/ci_test.py b/ci_test.py
index a264fb1d..ee4da17f 100755
--- a/ci_test.py
+++ b/ci_test.py
@@ -16,19 +16,22 @@
 import urllib.request
 import json
 import subprocess
+import shlex
 import sys
 import os
 
 
 def run_command(cmd, log_file=None):
-    print("Running: {}".format(cmd))
+    if isinstance(cmd, str):
+        cmd = shlex.split(cmd)
+    print("Running: {}".format(shlex.join(cmd)))
     sys.stdout.flush()
     if log_file:
         file = open(log_file, "w")
-        p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file)
     else:
-        p = subprocess.Popen(cmd, shell=True)
-      
+        file = None
+    p = subprocess.Popen(cmd, stdout=file, stderr=file)
+
     (output, err) = p.communicate()
     return p.wait()
 
@@ -62,22 +65,37 @@ def main():
     results = {}
     suite_status = True
     dockerfiles = get_dockerfiles()
+    root_dir = os.path.dirname(os.path.realpath(__file__))
     for dockerfile in dockerfiles:
-        docker_dir = os.path.dirname(os.path.realpath(__file__))
+        # Make sure everything is relative
+        dockerfile = os.path.relpath(os.path.realpath(dockerfile), root_dir)
+
+        docker_dir, docker_name = os.path.split(dockerfile)
+
         print("Testing {}".format(dockerfile))
         sys.stdout.flush()
-        log_file = dockerfile.replace(docker_dir,"").replace("/", "_")
+        log_file = dockerfile.replace("/", "_")
         log_file = "{}.log".format(log_file)
-        cmd = "docker build --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile)
+        cmd = [
+            'docker', 'build', '--no-cache=true',
+            '-f', dockerfile,
+            docker_dir
+        ]
         if "buildx" in dockerfile:
             # if "buildx" is part of the path, we want to use the new buildx build system and build
             # for both amd64 and arm64.
-            cmd = "docker buildx create --use"
-            run_command(cmd, log_file)
-            cmd = "docker buildx inspect --bootstrap"
-            run_command(cmd, log_file)
-            cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile)
-        status = run_command(cmd, log_file)
+            run_command("docker buildx create --use", log_file=log_file)
+            run_command("docker buildx inspect --bootstrap", log_file=log_file)
+
+            cmd = [
+                'docker', 'buildx', 'build',
+                '--platform', 'linux/arm64,linux/amd64',
+                '--no-cache=true',
+                '-f', dockerfile,
+                docker_dir
+            ]
+
+        status = run_command(cmd, log_file=log_file)
         results[dockerfile] = status
         if status != 0:
             suite_status = False
@@ -85,7 +103,9 @@ def main():
         else:
             results[dockerfile] = "PASSED"
 
-        cmd = "mv {log} {results}{log}".format(log=log_file, results=results[dockerfile])
+        cmd = [
+            'mv', log_file, results[dockerfile] + log_file
+        ]
         run_command(cmd)
         print("[{}] - {}".format(results[dockerfile], dockerfile))
         sys.stdout.flush()
diff --git a/swift-ci/sdks/static-linux/Dockerfile b/swift-ci/sdks/static-linux/Dockerfile
index e16e8c42..5253602b 100644
--- a/swift-ci/sdks/static-linux/Dockerfile
+++ b/swift-ci/sdks/static-linux/Dockerfile
@@ -24,6 +24,9 @@ ARG BORINGSSL_VERSION=fips-20220613
 ARG ICU_VERSION=maint/maint-69
 ARG ZLIB_VERSION=1.3.1
 
+# Architecture to build on (empty means x86-64)
+ARG OS_ARCH_SUFFIX=
+
 # ............................................................................
 
 # Install development tools
@@ -62,7 +65,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
     OS_MAJOR_VER=$OS_MAJOR_VER \
     OS_MINOR_VER=$OS_MINOR_VER \
     OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER \
-    SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER"
+    SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER$OS_ARCH_SUFFIX"
 
 COPY scripts/install-swift.sh /scripts/install-swift.sh
 RUN chmod ugo+x /scripts/install-swift.sh
diff --git a/swift-ci/sdks/static-linux/build b/swift-ci/sdks/static-linux/build
index 7bbc2bf0..6e984960 100755
--- a/swift-ci/sdks/static-linux/build
+++ b/swift-ci/sdks/static-linux/build
@@ -14,10 +14,25 @@
 #
 # ===----------------------------------------------------------------------===
 
-DOCKER=docker
+if [[ "$DOCKER" == "" ]]; then
+    DOCKER=docker
+fi
+
+case $(arch) in
+    arm64|aarch64)
+        OS_ARCH_SUFFIX=-aarch64
+        ;;
+    amd64|x86_64)
+        OS_ARCH_SUFFIX=
+        ;;
+    *)
+        echo "Unknown architecture $(arch)"
+        exit 1
+        ;;
+esac
 
 # Build the Docker image
-$(DOCKER) build -t static-swift-linux .
+$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX -t static-swift-linux .
 
 # Check-out the sources
 scripts/fetch-source.sh --clone-with-ssh --source-dir source
@@ -25,7 +40,7 @@ scripts/fetch-source.sh --clone-with-ssh --source-dir source
 mkdir -p products
 
 # Run the build
-$(DOCKER) run -it --rm \
+$DOCKER run -it --rm \
           -v ./source:/source \
           -v ./products:/products \
           static-swift-linux \
diff --git a/swift-ci/sdks/static-linux/scripts/build.sh b/swift-ci/sdks/static-linux/scripts/build.sh
index 798d5381..4012b6ac 100755
--- a/swift-ci/sdks/static-linux/scripts/build.sh
+++ b/swift-ci/sdks/static-linux/scripts/build.sh
@@ -144,6 +144,20 @@ while [ "$#" -gt 0 ]; do
     shift
 done
 
+# Work out the host architecture
+case $(arch) in
+    arm64|aarch64)
+        host_arch=arm64
+        ;;
+    amd64|x86_64)
+        host_arch=x86_64
+        ;;
+    *)
+        echo "Unknown host architecture $(arch)"
+        exit 1
+        ;;
+esac
+
 # Change the commas for spaces
 archs="${archs//,/ }"
 
@@ -662,7 +676,7 @@ EOF
         --compiler-vendor=apple \
         --bootstrapping hosttools \
         --build-linux-static --install-swift \
-        --stdlib-deployment-targets linux-x86_64,linux-static-$arch \
+        --stdlib-deployment-targets linux-$host_arch,linux-static-$arch \
         --build-stdlib-deployment-targets all \
         --musl-path=${build_dir}/sdk_root \
         --linux-static-arch=$arch \
@@ -724,6 +738,7 @@ EOF
           -DFOUNDATION_PATH_TO_LIBDISPATCH_BUILD=${build_dir}/$arch/dispatch \
           -D_SwiftFoundation_SourceDIR=${source_dir}/swift-project/swift-foundation \
           -D_SwiftFoundationICU_SourceDIR=${source_dir}/swift-project/swift-foundation-icu \
+          -D_SwiftCollections_SourceDIR=${source_dir}/swift-project/swift-collections \
           -DCMAKE_Swift_COMPILER_WORKS=YES \
           -Ddispatch_DIR=${build_dir}/$arch/dispatch/cmake/modules
 
diff --git a/swift-ci/sdks/static-linux/scripts/fetch-source.sh b/swift-ci/sdks/static-linux/scripts/fetch-source.sh
index edda203d..7eb5827a 100755
--- a/swift-ci/sdks/static-linux/scripts/fetch-source.sh
+++ b/swift-ci/sdks/static-linux/scripts/fetch-source.sh
@@ -164,7 +164,7 @@ cd swift
 # Get its dependencies
 header "Fetching Swift Dependencies"
 
-extra_args=--skip-history
+extra_args="--skip-history --all-repositories"
 if [[ $SWIFT_VERSION == scheme:* ]]; then
     utils/update-checkout ${clone_arg} --scheme ${SWIFT_VERSION#scheme:} ${extra_args}
 elif [[ $SWIFT_VERSION == tag:* ]]; then