diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index c974a814354b..3e5daae8778e 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -92,6 +92,14 @@ runs: readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true + - name: Spilt Binary Symbols + shell: bash + run: | + objcopy --only-keep-debug ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug + chmod 0644 ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug + strip --strip-debug --remove-section=.comment --remove-section=.note ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query + pushd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }} && objcopy --add-gnu-debuglink databend-query.debug databend-query && popd + # - name: Compress Binaries with UPX # if: env.BUILD_PROFILE == 'debug' # uses: crazy-max/ghaction-upx@v2 diff --git a/.github/actions/pack_binaries/action.yml b/.github/actions/pack_binaries/action.yml new file mode 100644 index 000000000000..64d3283b129b --- /dev/null +++ b/.github/actions/pack_binaries/action.yml @@ -0,0 +1,95 @@ +name: "Pack Binary" +description: "Pack releases binaries" +inputs: + target: + description: "Release target" + required: true + category: + description: "Release default/hdfs/udf/testsuite" + required: false + default: default + version: + description: "Release version" + required: true + +runs: + using: "composite" + steps: + - name: Download artifact + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ inputs.target }} + category: ${{ inputs.category }} + path: distro/bin + artifacts: metactl,meta,query,query.debug + - name: Pack Binaries + id: pack_binaries + shell: bash + run: | + target=${{ inputs.target }} + version=${{ inputs.version }} + case ${{ inputs.category }} in + default) + pkg_name="databend-${version}-${target}" + ;; + *) + pkg_name="databend-${{ inputs.category }}-${version}-${target}" + ;; + esac + mkdir -p distro/{bin,configs,systemd,scripts} + cp ./scripts/distribution/systemd/databend-* distro/systemd/ + cp ./scripts/distribution/configs/databend-* distro/configs/ + cp ./scripts/distribution/release-readme.txt distro/readme.txt + cp -r ./scripts/distribution/local-scripts/* distro/scripts/ + cp -r ./scripts/distribution/package-scripts/* distro/scripts/ + tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt + sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: post sha256 + uses: actions/upload-artifact@v4 + with: + name: sha256sums-${{ inputs.category }}-${{ inputs.target }} + path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt + retention-days: 1 + - name: post binaries + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz + path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz + retention-days: 1 + - name: Pack DBG Binaries + id: pack_dbg_binaries + shell: bash + run: | + target=${{ inputs.target }} + version=${{ inputs.version }} + case ${{ inputs.category }} in + default) + pkg_name="databend-${version}-${target}-dbg" + ;; + *) + pkg_name="databend-${{ inputs.category }}-${version}-${target}-dbg" + ;; + esac + mkdir -p distro/{bin,configs,systemd,scripts} + cp ./scripts/distribution/systemd/databend-* distro/systemd/ + cp ./scripts/distribution/configs/databend-* distro/configs/ + cp ./scripts/distribution/release-readme.txt distro/readme.txt + cp -r ./scripts/distribution/local-scripts/* distro/scripts/ + cp -r ./scripts/distribution/package-scripts/* distro/scripts/ + tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt + sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: post dbg sha256 + uses: actions/upload-artifact@v4 + with: + name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb + path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt + retention-days: 1 + - name: post dbg binaries + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz + path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz + retention-days: 1 diff --git a/.github/actions/pack_distribution/action.yml b/.github/actions/pack_distribution/action.yml new file mode 100644 index 000000000000..57eab18853e2 --- /dev/null +++ b/.github/actions/pack_distribution/action.yml @@ -0,0 +1,94 @@ +name: "Pack Deb" +description: "Pack releases deb" +inputs: + arch: + description: "Release arch" + required: true + packager: + description: "Release default/hdfs/udf/testsuite" + required: false + default: default + version: + description: "Release version" + required: true + +runs: + using: "composite" + steps: + - name: Install nfpm@latest + shell: bash + run: | + curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz + tar xf nfpm.tar.gz + sudo mv nfpm /usr/local/bin + sudo chmod a+x /usr/local/bin/nfpm + rm nfpm.tar.gz + - name: Get target + id: target + shell: bash + run: | + echo 'target=${{ inputs.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT + - name: Download artifacts + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ steps.target.outputs.target }} + category: default + artifacts: metactl,meta,query,query.debug + path: distro/bin + - name: Build Packages + shell: bash + id: build_packages + run: | + export name="databend" + export version="${{ inputs.version }}" + export path="distro" + case "${{ inputs.arch }}" in + x86_64) + export arch="amd64" + ;; + aarch64) + export arch="arm64" + ;; + esac + + deb_version=${version/-/.} + pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}.${{ inputs.packager }}" + mkdir -p distro/{bin,configs,systemd,scripts} + cp ./scripts/distribution/systemd/databend-* distro/systemd/ + cp ./scripts/distribution/configs/databend-* distro/configs/ + cp ./scripts/distribution/release-readme.txt distro/readme.txt + cp -r ./scripts/distribution/local-scripts/* distro/scripts/ + cp -r ./scripts/distribution/package-scripts/* distro/scripts/ + nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml) + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: Build dbg Packages + shell: bash + id: build_dbg_packages + run: | + export name="databend_dbg" + export version="${{ inputs.version }}" + export path="distro" + case "${{ inputs.arch }}" in + x86_64) + export arch="amd64" + ;; + aarch64) + export arch="arm64" + ;; + esac + + deb_version=${version/-/.} + pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}_dbg.${{ inputs.packager }}" + nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm-dbg.yaml) + echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT + - name: Update release to github + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Reference: https://cli.github.com/manual/gh_release_upload + run: | + version="${{ inputs.version }}" + # name looks like: `databend_0.8.144~nightly_amd64.deb` + gh release upload ${version} ${{ steps.build_packages.outputs.pkg_name }} --clobber + gh release upload ${version} ${{ steps.build_dbg_packages.outputs.pkg_name }} --clobber diff --git a/.github/actions/publish_binary/action.yml b/.github/actions/publish_binary/action.yml index f73fd02f419d..b9e3d470df9a 100644 --- a/.github/actions/publish_binary/action.yml +++ b/.github/actions/publish_binary/action.yml @@ -37,13 +37,19 @@ runs: shell: bash # Reference: https://cli.github.com/manual/gh_release_upload run: | - gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.* --clobber + gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber + if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then + gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.* --clobber + fi - name: Sync normal release to R2 shell: bash if: inputs.category == 'default' run: | aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress + if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then + aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress + fi gh api /repos/databendlabs/databend/tags > tags.json aws s3 cp ./tags.json s3://repo/databend/tags.json gh api /repos/databendlabs/databend/releases > releases.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 366a7c7a3d13..2382725e1278 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -203,40 +203,12 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ needs.create_release.outputs.sha }} - - name: Download artifact - uses: ./.github/actions/artifact_download + - name: Pack Binaries + uses: ./.github/actions/pack_binaries with: - sha: ${{ github.sha }} + version: ${{ needs.create_release.outputs.version }} target: ${{ matrix.target }} category: ${{ matrix.category }} - path: distro/bin - artifacts: metactl,meta,query - - name: Pack Binaries - run: | - target=${{ matrix.target }} - version=${{ needs.create_release.outputs.version }} - case ${{ matrix.category }} in - default) - pkg_name="databend-${version}-${target}" - ;; - *) - pkg_name="databend-${{ matrix.category }}-${version}-${target}" - ;; - esac - mkdir -p distro/{bin,configs,systemd,scripts} - cp ./scripts/distribution/systemd/databend-* distro/systemd/ - cp ./scripts/distribution/configs/databend-* distro/configs/ - cp ./scripts/distribution/release-readme.txt distro/readme.txt - cp -r ./scripts/distribution/local-scripts/* distro/scripts/ - cp -r ./scripts/distribution/package-scripts/* distro/scripts/ - tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt - sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt - - name: post sha256 - uses: actions/upload-artifact@v4 - with: - name: sha256sums-${{ matrix.category }}-${{ matrix.target }} - path: sha256-*.txt - retention-days: 1 - name: Publish Binaries uses: ./.github/actions/publish_binary env: @@ -455,55 +427,12 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ needs.create_release.outputs.sha }} - - name: Install nfpm@latest - run: | - curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz - tar xf nfpm.tar.gz - sudo mv nfpm /usr/local/bin - sudo chmod a+x /usr/local/bin/nfpm - rm nfpm.tar.gz - - name: Get target - id: target - run: | - echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT - - name: Download artifacts - uses: ./.github/actions/artifact_download + - name: Pack distribution + uses: ./.github/actions/pack_distribution with: - sha: ${{ github.sha }} - target: ${{ steps.target.outputs.target }} - category: default - artifacts: metactl,meta,query - path: distro/bin - - name: Build Packages - id: build_packages - run: | - export name="databend" - export version="${{ needs.create_release.outputs.version }}" - export path="distro" - case "${{ matrix.arch }}" in - x86_64) - export arch="amd64" - ;; - aarch64) - export arch="arm64" - ;; - esac - mkdir -p distro/{bin,configs,systemd,scripts} - cp ./scripts/distribution/systemd/databend-* distro/systemd/ - cp ./scripts/distribution/configs/databend-* distro/configs/ - cp ./scripts/distribution/release-readme.txt distro/readme.txt - cp -r ./scripts/distribution/local-scripts/* distro/scripts/ - cp -r ./scripts/distribution/package-scripts/* distro/scripts/ - nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml) - - name: Update release to github - shell: bash - env: - GH_TOKEN: ${{ github.token }} - # Reference: https://cli.github.com/manual/gh_release_upload - run: | - version="${{ needs.create_release.outputs.version }}" - # name looks like: `databend_0.8.144~nightly_amd64.deb` - gh release upload ${version} databend_*.${{ matrix.packager }} --clobber + version: ${{ needs.create_release.outputs.version }} + arch: ${{ matrix.arch }} + packager: ${{ matrix.packager }} # bindings_python: # if: inputs.stable diff --git a/scripts/distribution/nfpm-dbg.yaml b/scripts/distribution/nfpm-dbg.yaml new file mode 100644 index 000000000000..811f607cf171 --- /dev/null +++ b/scripts/distribution/nfpm-dbg.yaml @@ -0,0 +1,17 @@ +name: "${name}" +arch: "${arch}" +platform: "linux" +version: "${version}" +section: "database" +priority: "extra" +maintainer: "Databend Authors " +description: | + Databend is a powerful cloud data warehouse. Built for elasticity and efficiency. + Free and open. Also available in the cloud: https://app.databend.com +vendor: "Datafuse Labs" +homepage: "https://databend.com" +license: "Apache-2.0" +contents: + # Binaries + - src: ${path}/bin/databend-query.debug + dst: /usr/bin/databend-query.debug diff --git a/scripts/setup/dev_setup.sh b/scripts/setup/dev_setup.sh index 24a2ace0f21d..7f7b8b95d0dc 100755 --- a/scripts/setup/dev_setup.sh +++ b/scripts/setup/dev_setup.sh @@ -393,6 +393,28 @@ function install_libtiff { esac } +function install_binutils { + PACKAGE_MANAGER=$1 + + echo "==> installing binutils..." + + case "$PACKAGE_MANAGER" in + apt-get) + install_pkg binutils "$PACKAGE_MANAGER" + ;; + yum | dnf) + install_pkg binutils "$PACKAGE_MANAGER" + ;; + brew) + # skip + ;; + *) + echo "Unable to install binutils with package manager: $PACKAGE_MANAGER" + exit 1 + ;; + esac +} + function install_rustup { RUST_TOOLCHAIN=$1 @@ -624,6 +646,7 @@ if [[ "$INSTALL_BUILD_TOOLS" == "true" ]]; then install_python3 "$PACKAGE_MANAGER" install_sqlite3 "$PACKAGE_MANAGER" install_libtiff "$PACKAGE_MANAGER" + install_binutils "$PACKAGE_MANAGER" # Any call to cargo will make rustup install the correct toolchain cargo version