From 416614b17cef2edf0423ccb1c1edb695610c7838 Mon Sep 17 00:00:00 2001 From: past-due <30942300+past-due@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:26:57 -0500 Subject: [PATCH 1/3] [GitHub Actions] Use org.flatpak.Builder flatpak to better match Flathub infra --- .ci/flatpak/build.sh | 62 +++++++++++++++++------- .ci/flatpak/net.wz2100.wz2100.yaml.in | 1 + .github/workflows/CI_flatpak.yml | 68 ++++++++++----------------- 3 files changed, 71 insertions(+), 60 deletions(-) diff --git a/.ci/flatpak/build.sh b/.ci/flatpak/build.sh index 29841a8113f..85491267fbd 100755 --- a/.ci/flatpak/build.sh +++ b/.ci/flatpak/build.sh @@ -1,8 +1,35 @@ #!/bin/bash # This requires a bunch of environment variables to be set. See the CI workflow +USE_BUILDER_FLATPAK=true + +if [ "${USE_BUILDER_FLATPAK}" = true ]; then + FLATPAK_BUILDER_CMD="flatpak run --filesystem=/tmp org.flatpak.Builder" + FLATPAK_CMD="flatpak run --filesystem=/tmp --command=flatpak org.flatpak.Builder" + OSTREE_CMD="flatpak run --filesystem=/tmp --command=ostree org.flatpak.Builder" + APPSTREAMCLI_CMD="flatpak run --filesystem=/tmp --command=appstreamcli org.flatpak.Builder" +else + FLATPAK_BUILDER_CMD="flatpak-builder" + FLATPAK_CMD="flatpak" + OSTREE_CMD="ostree" + APPSTREAMCLI_CMD="appstreamcli" +fi + +echo "::group::flatpak-builder --version" +${FLATPAK_BUILDER_CMD} --version +echo "::endgroup::" +echo "::group::flatpak --version" +${FLATPAK_CMD} --version +echo "::endgroup::" +echo "::group::ostree --version" +${OSTREE_CMD} --version +echo "::endgroup::" +echo "::group::appstreamcli --version" +${APPSTREAMCLI_CMD} --version +echo "::endgroup::" + echo "::group::flatpak-builder" -flatpak-builder --repo=${WZ_FLATPAK_LOCAL_REPO_NAME} --disable-rofiles-fuse --force-clean --default-branch=${WZ_FLATPAK_BRANCH} --mirror-screenshots-url=${WZ_FLATPAK_MIRROR_SCREENSHOTS_URL} "${WZ_FLATPAK_BUILD_DIR}" ${WZ_FLATPAK_MANIFEST_PATH} +${FLATPAK_BUILDER_CMD} --repo=${WZ_FLATPAK_LOCAL_REPO_NAME} --disable-rofiles-fuse --force-clean --default-branch=${WZ_FLATPAK_BRANCH} --mirror-screenshots-url=${WZ_FLATPAK_MIRROR_SCREENSHOTS_URL} "${WZ_FLATPAK_BUILD_DIR}" ${WZ_FLATPAK_MANIFEST_PATH} echo "::endgroup::" if [[ "$WZ_FLATPAK_TARGET_ARCH" != "$WZ_FLATPAK_BUILD_ARCH" ]]; then @@ -11,7 +38,7 @@ if [[ "$WZ_FLATPAK_TARGET_ARCH" != "$WZ_FLATPAK_BUILD_ARCH" ]]; then # Create a new repository containing the commits for the cross target arch echo "::group::Creating new local repo for target arch: ${WZ_FLATPAK_LOCAL_REPO_NAME}" WZ_FLATPAK_LOCAL_REPO_NAME="${WZ_FLATPAK_TARGET_ARCH}-repo" - ostree init --mode archive-z2 --repo=${WZ_FLATPAK_LOCAL_REPO_NAME} + ${OSTREE_CMD} init --mode archive-z2 --repo=${WZ_FLATPAK_LOCAL_REPO_NAME} echo "::endgroup::" echo "::group::Rename commits to new target arch repo" @@ -20,14 +47,14 @@ if [[ "$WZ_FLATPAK_TARGET_ARCH" != "$WZ_FLATPAK_BUILD_ARCH" ]]; then do # Rename the commits to target arch echo "Processing: --src-ref=${i}/${WZ_FLATPAK_BUILD_ARCH}/${WZ_FLATPAK_BRANCH}" - flatpak build-commit-from --src-ref=${i}/${WZ_FLATPAK_BUILD_ARCH}/${WZ_FLATPAK_BRANCH} --src-repo=${SRC_LOCAL_REPO_NAME} \ + ${FLATPAK_CMD} build-commit-from --src-ref=${i}/${WZ_FLATPAK_BUILD_ARCH}/${WZ_FLATPAK_BRANCH} --src-repo=${SRC_LOCAL_REPO_NAME} \ ${WZ_FLATPAK_LOCAL_REPO_NAME} ${i}/${WZ_FLATPAK_TARGET_ARCH}/${WZ_FLATPAK_BRANCH} done echo "::endgroup::" fi echo "::group::flatpak build-bundle" -flatpak build-bundle --arch=${WZ_FLATPAK_TARGET_ARCH} ${WZ_FLATPAK_LOCAL_REPO_NAME} "${WZ_FLATPAK_BUNDLE}" --runtime-repo=https://flathub.org/repo/flathub.flatpakrepo ${WZ_FLATPAK_APPID} ${WZ_FLATPAK_BRANCH} +${FLATPAK_CMD} build-bundle --arch=${WZ_FLATPAK_TARGET_ARCH} ${WZ_FLATPAK_LOCAL_REPO_NAME} "${WZ_FLATPAK_BUNDLE}" --runtime-repo=https://flathub.org/repo/flathub.flatpakrepo ${WZ_FLATPAK_APPID} ${WZ_FLATPAK_BRANCH} echo "::endgroup::" echo "Generated .flatpak: \"${WZ_FLATPAK_BUNDLE}\"" @@ -44,14 +71,8 @@ fi if [ -f "${WZ_FLATPAK_APPSTREAM_PATH}" ]; then echo "::group::Validating appstream" - if [ -x "$(command -v appstreamcli)" ]; then - echo "appstreamcli validate ${WZ_FLATPAK_APPSTREAM_PATH}" - appstreamcli validate "${WZ_FLATPAK_APPSTREAM_PATH}" - else - # Older utility fallback - echo "appstream-util validate ${WZ_FLATPAK_APPSTREAM_PATH}" - appstream-util validate "${WZ_FLATPAK_APPSTREAM_PATH}" - fi + echo "appstreamcli validate ${WZ_FLATPAK_APPSTREAM_PATH}" + ${APPSTREAMCLI_CMD} validate "${WZ_FLATPAK_APPSTREAM_PATH}" echo "::endgroup::" else echo "::warning ::Could not find appstream file to validate?" @@ -63,18 +84,25 @@ test -f "${WZ_FLATPAK_BUILD_PATH}/app-info/xmls/${WZ_FLATPAK_APPID}.xml.gz" || { echo "::endgroup::" echo "::group::Check screenshots" -if [ ! -d "${WZ_FLATPAK_BUILD_DIR}/screenshots" ]; then +if [ ! -d "${WZ_FLATPAK_BUILD_PATH}/app-info/media" ]; then echo "::notice ::Screenshots not mirrored by flatpak-builder?" else echo "Found screenshots:" - find "${WZ_FLATPAK_BUILD_DIR}/screenshots" -type f + find "${WZ_FLATPAK_BUILD_PATH}/app-info/media" -type f fi echo "::endgroup::" -echo "::group::Commit screenshots to the OSTree repository" -if [ -d "${WZ_FLATPAK_BUILD_DIR}/screenshots" ]; then - ostree commit --repo=${WZ_FLATPAK_LOCAL_REPO_NAME} --canonical-permissions --branch=screenshots/${WZ_FLATPAK_TARGET_ARCH} "${WZ_FLATPAK_BUILD_DIR}/screenshots" +echo "::group::Commit screenshots to the OSTree repository --branch=screenshots/${WZ_FLATPAK_TARGET_ARCH}" +if [ -d "${WZ_FLATPAK_BUILD_PATH}/app-info/media" ]; then + ${OSTREE_CMD} commit --repo=${WZ_FLATPAK_LOCAL_REPO_NAME} --canonical-permissions --branch=screenshots/${WZ_FLATPAK_TARGET_ARCH} "${WZ_FLATPAK_BUILD_PATH}/app-info/media" else echo "::warning ::Screenshots not added to OSTree repository" fi echo "::endgroup::" + +# Output final WZ_FLATPAK_LOCAL_REPO_NAME in GitHub Actions +if [ "${GITHUB_ACTIONS}" == "true" ]; then + echo "Running on GitHub Actions - outputting final WZ_FLATPAK_LOCAL_REPO_NAME" + echo "WZ_FLATPAK_LOCAL_REPO_NAME=${WZ_FLATPAK_LOCAL_REPO_NAME}" + echo "WZ_FLATPAK_LOCAL_REPO_NAME=${WZ_FLATPAK_LOCAL_REPO_NAME}" >> $GITHUB_OUTPUT +fi diff --git a/.ci/flatpak/net.wz2100.wz2100.yaml.in b/.ci/flatpak/net.wz2100.wz2100.yaml.in index ea82ae588ef..31dae1a4eaf 100755 --- a/.ci/flatpak/net.wz2100.wz2100.yaml.in +++ b/.ci/flatpak/net.wz2100.wz2100.yaml.in @@ -95,6 +95,7 @@ modules: buildsystem: cmake-ninja builddir: true build-options: + strip: true no-debuginfo: true # Always build for host - do not include WZ_CROSS_BUILD_OPTIONS cleanup: diff --git a/.github/workflows/CI_flatpak.yml b/.github/workflows/CI_flatpak.yml index b0ce5fabf64..0187eba8628 100644 --- a/.github/workflows/CI_flatpak.yml +++ b/.github/workflows/CI_flatpak.yml @@ -48,9 +48,6 @@ jobs: WZ_FLATPAK_TARGET_ARCH: ${{ matrix.flatpak-arch }} WZ_FLATPAK_BUILD_ARCH: x86_64 WZ_FREEDESKTOP_RUNTIME_VERSION: 23.08 # should match runtime-version in .ci/flatpak/net.wz2100.wz2100.yaml.in - container: - image: fedora:latest - options: --privileged outputs: # Needed by the release job - despite this being a matrix job, this should be the same for all, so we can allow whatever is last to persist it WZ_GITHUB_REF: ${{ steps.checkout-config.outputs.WZ_GITHUB_REF }} @@ -59,34 +56,15 @@ jobs: steps: - name: Install Dependencies run: | - cat /etc/fedora-release - - echo "::group::dnf update" - dnf update -y - echo "::endgroup::" - echo "::group::dnf install dependencies" - dnf install -y cmake git git-lfs dbus-daemon flatpak flatpak-builder python3-aiohttp python3-tenacity python3-gobject ccache zstd curl appstream + echo "::group::apt install dependencies" + sudo apt install flatpak python3-aiohttp python3-tenacity dbus-daemon curl echo "::endgroup::" echo "::group::flatpak remote-add" - flatpak remote-add --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo - flatpak remote-add --if-not-exists flathub-beta https://flathub.org/beta-repo/flathub-beta.flatpakrepo - flatpak remote-add --if-not-exists gnome-nightly https://nightly.gnome.org/gnome-nightly.flatpakrepo - echo "::endgroup::" - - echo "::group::fetch flat-manager-client" - curl -Lf https://raw.githubusercontent.com/flatpak/flat-manager/master/flat-manager-client --output /usr/bin/flat-manager-client - chmod +x /usr/bin/flat-manager-client + flatpak --user remote-add --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo + flatpak --user remote-add --if-not-exists flathub-beta https://flathub.org/beta-repo/flathub-beta.flatpakrepo + flatpak --user remote-add --if-not-exists gnome-nightly https://nightly.gnome.org/gnome-nightly.flatpakrepo echo "::endgroup::" - - name: Install QEMU deps - if: ${{ matrix.arch != 'x86_64' }} - run: | - dnf -y install docker - - name: Set up QEMU - if: ${{ matrix.arch != 'x86_64' }} - uses: docker/setup-qemu-action@v2 - with: - platforms: ${{ matrix.arch }} - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -201,19 +179,19 @@ jobs: - name: Install Flatpak manifest deps run: | # Build SDK - flatpak --system install -y --noninteractive flathub org.freedesktop.Sdk//${WZ_FREEDESKTOP_RUNTIME_VERSION} + flatpak install --user -y --noninteractive flathub org.freedesktop.Sdk//${WZ_FREEDESKTOP_RUNTIME_VERSION} # Target runtime - flatpak --system install -y --noninteractive flathub org.freedesktop.Platform/${WZ_FLATPAK_TARGET_ARCH}/${WZ_FREEDESKTOP_RUNTIME_VERSION} + flatpak install --user -y --noninteractive flathub org.freedesktop.Platform/${WZ_FLATPAK_TARGET_ARCH}/${WZ_FREEDESKTOP_RUNTIME_VERSION} if [[ "$WZ_FLATPAK_TARGET_ARCH" != "$WZ_FLATPAK_BUILD_ARCH" ]]; then # Cross compiler - flatpak --system install -y --noninteractive flathub org.freedesktop.Sdk.Extension.toolchain-${WZ_FLATPAK_TARGET_ARCH}//${WZ_FREEDESKTOP_RUNTIME_VERSION} + flatpak install --user -y --noninteractive flathub org.freedesktop.Sdk.Extension.toolchain-${WZ_FLATPAK_TARGET_ARCH}//${WZ_FREEDESKTOP_RUNTIME_VERSION} # SDK For target runtime - flatpak --system install -y --noninteractive flathub org.freedesktop.Sdk.Compat.${WZ_FLATPAK_TARGET_ARCH}//${WZ_FREEDESKTOP_RUNTIME_VERSION} + flatpak install --user -y --noninteractive flathub org.freedesktop.Sdk.Compat.${WZ_FLATPAK_TARGET_ARCH}//${WZ_FREEDESKTOP_RUNTIME_VERSION} fi # Builder (which includes flatpak-builder-lint) - flatpak install flathub -y --noninteractive org.flatpak.Builder + flatpak install --user flathub -y --noninteractive org.flatpak.Builder - name: Build Flatpak id: build env: @@ -223,9 +201,7 @@ jobs: SENTRY_IO_DSN: '${{ secrets.CRASHREPORTING_SENTRY_IO_DSN }}' DISCORD_RPC_APPID: '${{ secrets.DISCORD_RPC_APPID }}' run: | - . src/.ci/flatpak/build.sh - echo "WZ_FLATPAK_LOCAL_REPO_NAME=${WZ_FLATPAK_LOCAL_REPO_NAME}" - echo "WZ_FLATPAK_LOCAL_REPO_NAME=${WZ_FLATPAK_LOCAL_REPO_NAME}" >> $GITHUB_OUTPUT + dbus-run-session -- bash src/.ci/flatpak/build.sh - name: Output generated files listing shell: bash working-directory: flatpak_app/files @@ -236,12 +212,14 @@ jobs: env: WZ_FLATPAK_LOCAL_REPO_NAME: ${{ steps.build.outputs.WZ_FLATPAK_LOCAL_REPO_NAME }} run: | - # Start a dbus session within the container to allow flatpaks to run - # See: https://discussion.fedoraproject.org/t/cannot-run-flatpaks-in-a-fedora-container/73867/2 - mkdir /run/dbus - dbus-send --system /org/freedesktop/DBus org.freedesktop.DBus || dbus-daemon --system --fork - - flatpak run --command=flatpak-builder-lint org.flatpak.Builder --exceptions repo "${WZ_FLATPAK_LOCAL_REPO_NAME}" + WZ_FLATPAK_BUILD_PATH="${WZ_FLATPAK_BUILD_DIR}/files/share" + echo "::group::.metainfo.xml file" + cat "${WZ_FLATPAK_BUILD_PATH}/metainfo/${WZ_FLATPAK_APPID}.metainfo.xml" + echo "::endgroup::" + echo "::group::.desktop file" + cat "${WZ_FLATPAK_BUILD_PATH}/applications/${WZ_FLATPAK_APPID}.desktop" + echo "::endgroup::" + dbus-run-session flatpak run --command=flatpak-builder-lint org.flatpak.Builder --exceptions repo "${WZ_FLATPAK_LOCAL_REPO_NAME}" echo "flatpak-builder-lint - OK" ############################################################### # Upload the .flatpak and debugsymbols artifacts @@ -264,15 +242,19 @@ jobs: shell: bash run: | mkdir output_debug_dependency_snapshot + # Get path to runtime files + TARGET_FREEDESKTOP_RUNTIME_PATH="$(flatpak info --show-location org.freedesktop.Platform/${WZ_FLATPAK_TARGET_ARCH}/${WZ_FREEDESKTOP_RUNTIME_VERSION})" + echo "TARGET_FREEDESKTOP_RUNTIME_PATH=${TARGET_FREEDESKTOP_RUNTIME_PATH}" + export TARGET_FREEDESKTOP_RUNTIME_PATH=${TARGET_FREEDESKTOP_RUNTIME_PATH} set +e # Note: These libraries will be from the runtime that is installed at the time of the build ldd flatpak_app/files/bin/warzone2100 | grep -E -v 'linux-vdso|ld-linux-' | awk 'NF == 4 { system("echo " $1) }' ret="$?" if [ $ret -eq 0 ]; then - ldd flatpak_app/files/bin/warzone2100 | grep -E -v 'linux-vdso|ld-linux-' | awk 'NF == 4 { system("cp /var/lib/flatpak/runtime/org.freedesktop.Platform/${WZ_FLATPAK_TARGET_ARCH}/${WZ_FREEDESKTOP_RUNTIME_VERSION}/active/files/lib/${WZ_FLATPAK_TARGET_ARCH}-linux-gnu/" $1 " output_debug_dependency_snapshot") }' + ldd flatpak_app/files/bin/warzone2100 | grep -E -v 'linux-vdso|ld-linux-' | awk 'NF == 4 { system("cp ${TARGET_FREEDESKTOP_RUNTIME_PATH}/files/lib/${WZ_FLATPAK_TARGET_ARCH}-linux-gnu/" $1 " output_debug_dependency_snapshot") }' else # ldd fails for non-native architecture, so use objdump for cross-compiled builds - objdump -p flatpak_app/files/bin/warzone2100 | grep NEEDED | grep -E -v 'linux-vdso|ld-linux-' | awk 'NF == 2 { system("echo " $2 " && cp /var/lib/flatpak/runtime/org.freedesktop.Platform/${WZ_FLATPAK_TARGET_ARCH}/${WZ_FREEDESKTOP_RUNTIME_VERSION}/active/files/lib/${WZ_FLATPAK_TARGET_ARCH}-linux-gnu/" $2 " output_debug_dependency_snapshot") }' + objdump -p flatpak_app/files/bin/warzone2100 | grep NEEDED | grep -E -v 'linux-vdso|ld-linux-' | awk 'NF == 2 { system("echo " $2 " && cp ${TARGET_FREEDESKTOP_RUNTIME_PATH}/files/lib/${WZ_FLATPAK_TARGET_ARCH}-linux-gnu/" $2 " output_debug_dependency_snapshot") }' fi - name: Upload additional debug dependency info uses: actions/upload-artifact@v4 From c3bc55fd1fabe5cb1435b195dfd26eace1ed4c95 Mon Sep 17 00:00:00 2001 From: past-due <30942300+past-due@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:50:53 -0500 Subject: [PATCH 2/3] [Flatpak] Update libsodium to 1.0.19 --- .ci/flatpak/net.wz2100.wz2100.yaml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/flatpak/net.wz2100.wz2100.yaml.in b/.ci/flatpak/net.wz2100.wz2100.yaml.in index 31dae1a4eaf..4885438578c 100755 --- a/.ci/flatpak/net.wz2100.wz2100.yaml.in +++ b/.ci/flatpak/net.wz2100.wz2100.yaml.in @@ -163,8 +163,8 @@ modules: @WZ_AUTOTOOLS_CROSS_CONFIG_OPTIONS@ sources: - type: archive - url: https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz - sha256: 6f504490b342a4f8a4c4a02fc9b866cbef8622d5df4e5452b46be121e46636c1 + url: https://github.com/jedisct1/libsodium/releases/download/1.0.19-RELEASE/libsodium-1.0.19.tar.gz + sha256: 018d79fe0a045cca07331d37bd0cb57b2e838c51bc48fd837a1472e50068bbea - name: warzone2100 buildsystem: cmake-ninja From 991054124d7af8ade3d97ecc74df54a8615fd49d Mon Sep 17 00:00:00 2001 From: past-due <30942300+past-due@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:10:34 -0500 Subject: [PATCH 3/3] [Flatpak] Update PhysFS (to pick up CMake fixes) --- .ci/flatpak/net.wz2100.wz2100.yaml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/flatpak/net.wz2100.wz2100.yaml.in b/.ci/flatpak/net.wz2100.wz2100.yaml.in index 4885438578c..8ebc6063cd1 100755 --- a/.ci/flatpak/net.wz2100.wz2100.yaml.in +++ b/.ci/flatpak/net.wz2100.wz2100.yaml.in @@ -139,8 +139,8 @@ modules: @WZ_CMAKE_CROSS_CONFIG_OPTIONS@ sources: - type: archive - url: https://github.com/icculus/physfs/archive/eb3383b532c5f74bfeb42ec306ba2cf80eed988c.zip # PhysFS 3.2.0 - sha512: 1068b1e0636b478f0d7910303e94d109923b2e4d809fa7a00232ba3733a4cd39c28d17851ff946859c431d7d7febdd229237cc10176eb5d5d19b49c887919fd8 + url: https://github.com/icculus/physfs/archive/741c4be358cd87366af1387d1e04e3f7d4f50c1e.zip # PhysFS 3.2.0+ + sha512: 56bfe808e28d7de6b9f242ca3ce81d08439480499c7d5901f78a3ec8ea75c55b71958515db015f101bb6b884a0a0f1140fab1a7222aff0bbe28555ebf378807f - name: fribidi buildsystem: meson