From 43a9b091921dd928ce6be357a0fbeb2332833683 Mon Sep 17 00:00:00 2001 From: jacekwegr Date: Tue, 3 Dec 2024 09:56:50 +0100 Subject: [PATCH 1/5] Add building `arm64` packages to CI pipeline --- .circleci/template.yml | 66 ++++++++++++++++++++++++++ tools/pkg/scripts/deb/build_package.sh | 8 +++- tools/pkg/scripts/rpm/build_package.sh | 25 +++++++++- 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index e528942dd8f..9879ef4a6c6 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -749,66 +749,132 @@ workflows: platform: debian-bookworm context: mongooseim-org filters: *all_tags_and_master + - package: + name: debian-bookworm-arm64 + executor: otp_27_arm64 + platform: debian-bookworm + context: mongooseim-org + filters: *all_tags_and_master - package: name: debian-bullseye executor: otp_27 platform: debian-bullseye context: mongooseim-org filters: *all_tags_and_master + - package: + name: debian-bullseye-arm64 + executor: otp_27_arm64 + platform: debian-bullseye + context: mongooseim-org + filters: *all_tags_and_master - package: name: debian-buster executor: otp_27 platform: debian-buster context: mongooseim-org filters: *all_tags_and_master + - package: + name: debian-buster-arm64 + executor: otp_27_arm64 + platform: debian-buster + context: mongooseim-org + filters: *all_tags_and_master - package: name: ubuntu-oracular executor: otp_27 platform: ubuntu-oracular context: mongooseim-org filters: *all_tags_and_master + - package: + name: ubuntu-oracular-arm64 + executor: otp_27_arm64 + platform: ubuntu-oracular + context: mongooseim-org + filters: *all_tags_and_master - package: name: ubuntu-noble executor: otp_27 platform: ubuntu-noble context: mongooseim-org filters: *all_tags_and_master + - package: + name: ubuntu-noble-arm64 + executor: otp_27_arm64 + platform: ubuntu-noble + context: mongooseim-org + filters: *all_tags_and_master - package: name: ubuntu-jammy executor: otp_27 platform: ubuntu-jammy context: mongooseim-org filters: *all_tags_and_master + - package: + name: ubuntu-jammy-arm64 + executor: otp_27_arm64 + platform: ubuntu-jammy + context: mongooseim-org + filters: *all_tags_and_master - package: name: ubuntu-focal executor: otp_27 platform: ubuntu-focal context: mongooseim-org filters: *all_tags_and_master + - package: + name: ubuntu-focal-arm64 + executor: otp_27_arm64 + platform: ubuntu-focal + context: mongooseim-org + filters: *all_tags_and_master - package: name: rockylinux-9 executor: otp_27 platform: rockylinux-9 context: mongooseim-org filters: *all_tags_and_master + - package: + name: rockylinux-9-arm64 + executor: otp_27_arm64 + platform: rockylinux-9 + context: mongooseim-org + filters: *all_tags_and_master - package: name: rockylinux-8 executor: otp_27 platform: rockylinux-8 context: mongooseim-org filters: *all_tags_and_master + - package: + name: rockylinux-8-arm64 + executor: otp_27_arm64 + platform: rockylinux-8 + context: mongooseim-org + filters: *all_tags_and_master - package: name: almalinux-9 executor: otp_27 platform: almalinux-9 context: mongooseim-org filters: *all_tags_and_master + - package: + name: almalinux-9-arm64 + executor: otp_27_arm64 + platform: almalinux-9 + context: mongooseim-org + filters: *all_tags_and_master - package: name: almalinux-8 executor: otp_27 platform: almalinux-8 context: mongooseim-org filters: *all_tags_and_master + - package: + name: almalinux-8-arm64 + executor: otp_27_arm64 + platform: almalinux-8 + context: mongooseim-org + filters: *all_tags_and_master # ======== BASE DOCKER BUILDS ======== - build_in_docker: name: otp_27_docker diff --git a/tools/pkg/scripts/deb/build_package.sh b/tools/pkg/scripts/deb/build_package.sh index eb08393763a..f366a84e483 100755 --- a/tools/pkg/scripts/deb/build_package.sh +++ b/tools/pkg/scripts/deb/build_package.sh @@ -5,7 +5,8 @@ version=$1 revision=$2 otp_version=$3 -arch="amd64" +arch=$(dpkg --print-architecture) +echo "Detected package architecture: $arch" cd ~/mongooseim @@ -40,6 +41,11 @@ sed -i "s#@DATE@#${date}#g" mongooseim/DEBIAN/changelog chown $USER:$USER -R mongooseim dpkg --build mongooseim ./ +if [[ $(dpkg-deb -f mongooseim_*.deb Architecture) != $arch ]]; then + echo "Error: The package was not built for ${arch}." + exit 1 +fi + source /etc/os-release os=$ID os_version=$VERSION_CODENAME diff --git a/tools/pkg/scripts/rpm/build_package.sh b/tools/pkg/scripts/rpm/build_package.sh index 719bb1d713a..d85fe0b5d13 100755 --- a/tools/pkg/scripts/rpm/build_package.sh +++ b/tools/pkg/scripts/rpm/build_package.sh @@ -5,8 +5,23 @@ version=$1 revision=$2 otp_version=$3 -arch="x86_64" -package_name_arch="amd64" +arch=$(uname -m) + +case "$arch" in + x86_64) + package_name_arch="amd64" + ;; + aarch64) + package_name_arch="arm64" + ;; + *) + echo "Unsupported architecture: $arch" + exit 1 + ;; +esac + +echo "Detected architecture: $arch" +echo "Package architecture: $package_name_arch" rpmbuild -bb \ @@ -15,6 +30,12 @@ rpmbuild -bb \ --define "architecture ${arch}" \ ~/rpmbuild/SPECS/mongooseim.spec +if [[ $(rpm -qp --qf "%{ARCH}" \ + ~/rpmbuild/RPMS/${arch}/mongooseim-${version}-${revision}.${arch}.rpm) != $arch ]]; then + echo "Error: The package was not built for ${arch}." + exit 1 +fi + source /etc/os-release os=$ID os_version=$VERSION_ID From ea424bc830d5a82eb03eaf5a09f82e73f5a00b9b Mon Sep 17 00:00:00 2001 From: jacekwegr Date: Thu, 5 Dec 2024 14:03:04 +0100 Subject: [PATCH 2/5] Install `debsigs` package through repository --- tools/pkg/Dockerfile_deb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/pkg/Dockerfile_deb b/tools/pkg/Dockerfile_deb index 92faf998a4e..ad8b821a073 100644 --- a/tools/pkg/Dockerfile_deb +++ b/tools/pkg/Dockerfile_deb @@ -8,15 +8,23 @@ FROM $builder_image AS builder # Install build deps ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -RUN apt-get install -y locales git make zlib1g-dev unixodbc-dev gcc g++ libssl-dev curl gpg wget gnupg +RUN apt-get install -y locales git make zlib1g-dev unixodbc-dev gcc g++ libssl-dev curl gpg # The signing script requires debsigs version 0.2 or higher, which is unavailable in # package repositories of Ubuntu versions earlier than 24.10 and Debian versions earlier than 13. # TODO: Switch to installing debsigs via apt once support for these older versions is dropped. -RUN wget http://ftp.de.debian.org/debian/pool/main/d/debsigs/debsigs_0.2.2-1_all.deb && \ - dpkg -i debsigs_0.2.2-1_all.deb && \ - rm debsigs_0.2.2-1_all.deb && \ - which debsigs +RUN distro=$(grep ^ID= /etc/os-release | cut -d= -f2 | tr -d '"') && \ + if [ "$distro" = "ubuntu" ]; then \ + echo "deb http://archive.ubuntu.com/ubuntu oracular main restricted universe multiverse" >> /etc/apt/sources.list && \ + printf "Package: debsigs\nPin: release n=oracular\nPin-Priority: 990\n" > /etc/apt/preferences.d/debsigs; \ + elif [ "$distro" = "debian" ]; then \ + echo "deb http://deb.debian.org/debian trixie main" >> /etc/apt/sources.list && \ + printf "Package: debsigs\nPin: release n=trixie\nPin-Priority: 990\n" > /etc/apt/preferences.d/debsigs; \ + fi && \ + apt-get update && \ + apt-get install -y --no-install-recommends debsigs && \ + sed -i '/oracular/d' /etc/apt/sources.list && \ + sed -i '/trixie/d' /etc/apt/sources.list ARG erlang_version From 4da7991f534d90d8301bb2e38a28fa624aecd4fe Mon Sep 17 00:00:00 2001 From: jacekwegr Date: Thu, 5 Dec 2024 14:43:12 +0100 Subject: [PATCH 3/5] Add package architecture verification script --- .circleci/template.yml | 6 ++++- tools/pkg/scripts/deb/build_package.sh | 6 ----- tools/pkg/scripts/rpm/build_package.sh | 10 ------- tools/pkg/verify_arch.sh | 37 ++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 17 deletions(-) create mode 100755 tools/pkg/verify_arch.sh diff --git a/.circleci/template.yml b/.circleci/template.yml index 9879ef4a6c6..6e5706cc265 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -707,6 +707,10 @@ jobs: name: Build package command: | ./tools/test.sh -p pkg -s false + - run: + name: Verify package architecture + command: | + ./tools/pkg/verify_arch.sh - when: condition: matches: @@ -725,7 +729,7 @@ filters: - &all_tags_and_master <<: *all_tags branches: - only: master + only: ci-build-packages-for-arm64 workflows: version: 2 diff --git a/tools/pkg/scripts/deb/build_package.sh b/tools/pkg/scripts/deb/build_package.sh index f366a84e483..9e7d4af1936 100755 --- a/tools/pkg/scripts/deb/build_package.sh +++ b/tools/pkg/scripts/deb/build_package.sh @@ -6,7 +6,6 @@ revision=$2 otp_version=$3 arch=$(dpkg --print-architecture) -echo "Detected package architecture: $arch" cd ~/mongooseim @@ -41,11 +40,6 @@ sed -i "s#@DATE@#${date}#g" mongooseim/DEBIAN/changelog chown $USER:$USER -R mongooseim dpkg --build mongooseim ./ -if [[ $(dpkg-deb -f mongooseim_*.deb Architecture) != $arch ]]; then - echo "Error: The package was not built for ${arch}." - exit 1 -fi - source /etc/os-release os=$ID os_version=$VERSION_CODENAME diff --git a/tools/pkg/scripts/rpm/build_package.sh b/tools/pkg/scripts/rpm/build_package.sh index d85fe0b5d13..6cc831c82bc 100755 --- a/tools/pkg/scripts/rpm/build_package.sh +++ b/tools/pkg/scripts/rpm/build_package.sh @@ -20,22 +20,12 @@ case "$arch" in ;; esac -echo "Detected architecture: $arch" -echo "Package architecture: $package_name_arch" - - rpmbuild -bb \ --define "version ${version}" \ --define "release ${revision}" \ --define "architecture ${arch}" \ ~/rpmbuild/SPECS/mongooseim.spec -if [[ $(rpm -qp --qf "%{ARCH}" \ - ~/rpmbuild/RPMS/${arch}/mongooseim-${version}-${revision}.${arch}.rpm) != $arch ]]; then - echo "Error: The package was not built for ${arch}." - exit 1 -fi - source /etc/os-release os=$ID os_version=$VERSION_ID diff --git a/tools/pkg/verify_arch.sh b/tools/pkg/verify_arch.sh new file mode 100755 index 00000000000..f8eb577e84b --- /dev/null +++ b/tools/pkg/verify_arch.sh @@ -0,0 +1,37 @@ +executor_arch=$(uname -m) +case "$executor_arch" in + x86_64) + expected_arch="amd64" + ;; + aarch64) + expected_arch="arm64" + ;; + *) + echo "Unsupported executor architecture: $executor_arch" + exit 1 + ;; +esac + +echo "Executor architecture detected: $expected_arch" + +package_file=$(find tools/pkg/packages -name "*.deb" -o -name "*.rpm") +if [ -z "$package_file" ]; then + echo "No package found in the output directory." + exit 1 +fi + +if [[ $package_file == *.deb ]]; then + actual_arch=$(dpkg --info "$package_file" | grep Architecture | awk '{print $2}') +elif [[ $package_file == *.rpm ]]; then + actual_arch=$(rpm -q --info "$package_file" | grep Architecture | awk '{print $3}') +else + echo "Unknown package type: $package_file" + exit 1 +fi + +if [ "$expected_arch" != "$actual_arch" ]; then + echo "Architecture mismatch: expected $expected_arch but got $actual_arch" + exit 1 +fi + +echo "Package architecture verified: $actual_arch" From 44d10a5612bf6a228528675709dc7c316c5815d4 Mon Sep 17 00:00:00 2001 From: jacekwegr Date: Thu, 5 Dec 2024 15:25:34 +0100 Subject: [PATCH 4/5] Fix the package scripts to make them work on CI --- .circleci/template.yml | 1 + tools/pkg/Dockerfile_deb | 11 ++++++++--- tools/pkg/verify_arch.sh | 10 +++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index 6e5706cc265..5d661a073f3 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -710,6 +710,7 @@ jobs: - run: name: Verify package architecture command: | + ./tools/circle-install-packages.sh rpm ./tools/pkg/verify_arch.sh - when: condition: diff --git a/tools/pkg/Dockerfile_deb b/tools/pkg/Dockerfile_deb index ad8b821a073..2b5ca060b52 100644 --- a/tools/pkg/Dockerfile_deb +++ b/tools/pkg/Dockerfile_deb @@ -12,10 +12,15 @@ RUN apt-get install -y locales git make zlib1g-dev unixodbc-dev gcc g++ libssl-d # The signing script requires debsigs version 0.2 or higher, which is unavailable in # package repositories of Ubuntu versions earlier than 24.10 and Debian versions earlier than 13. -# TODO: Switch to installing debsigs via apt once support for these older versions is dropped. -RUN distro=$(grep ^ID= /etc/os-release | cut -d= -f2 | tr -d '"') && \ +# TODO: Remove custom repo setup once older distro support ends. +RUN arch=$(dpkg --print-architecture) && \ + distro=$(grep ^ID= /etc/os-release | cut -d= -f2 | tr -d '"') && \ if [ "$distro" = "ubuntu" ]; then \ - echo "deb http://archive.ubuntu.com/ubuntu oracular main restricted universe multiverse" >> /etc/apt/sources.list && \ + if [ "$arch" = "arm64" ]; then \ + echo "deb http://ports.ubuntu.com/ubuntu-ports oracular main restricted universe multiverse" >> /etc/apt/sources.list; \ + else \ + echo "deb http://archive.ubuntu.com/ubuntu oracular main restricted universe multiverse" >> /etc/apt/sources.list; \ + fi && \ printf "Package: debsigs\nPin: release n=oracular\nPin-Priority: 990\n" > /etc/apt/preferences.d/debsigs; \ elif [ "$distro" = "debian" ]; then \ echo "deb http://deb.debian.org/debian trixie main" >> /etc/apt/sources.list && \ diff --git a/tools/pkg/verify_arch.sh b/tools/pkg/verify_arch.sh index f8eb577e84b..479cfe6911d 100755 --- a/tools/pkg/verify_arch.sh +++ b/tools/pkg/verify_arch.sh @@ -23,7 +23,15 @@ fi if [[ $package_file == *.deb ]]; then actual_arch=$(dpkg --info "$package_file" | grep Architecture | awk '{print $2}') elif [[ $package_file == *.rpm ]]; then - actual_arch=$(rpm -q --info "$package_file" | grep Architecture | awk '{print $3}') + actual_arch=$(rpm -qi "$package_file" | grep Architecture | awk '{print $2}') + if [ "$actual_arch" == "x86_64" ]; then + actual_arch="amd64" + elif [ "$actual_arch" == "aarch64" ]; then + actual_arch="arm64" + else + echo "Unable to determine architecture from file output." + exit 1 + fi else echo "Unknown package type: $package_file" exit 1 From 431357ecd39ea26d3a4d158df5aa68d1bee2162c Mon Sep 17 00:00:00 2001 From: jacekwegr Date: Fri, 6 Dec 2024 10:33:34 +0100 Subject: [PATCH 5/5] Change branch back to master for building packages --- .circleci/template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index 5d661a073f3..008089dccda 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -730,7 +730,7 @@ filters: - &all_tags_and_master <<: *all_tags branches: - only: ci-build-packages-for-arm64 + only: master workflows: version: 2