Skip to content

Commit

Permalink
chore(query): spilt binary symbol (#16861)
Browse files Browse the repository at this point in the history
* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* Update action.yml

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol

* chore(query): spilt binary symbol
  • Loading branch information
zhang2014 authored Nov 26, 2024
1 parent c9b1a82 commit 7b98a13
Show file tree
Hide file tree
Showing 7 changed files with 252 additions and 80 deletions.
8 changes: 8 additions & 0 deletions .github/actions/build_linux/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ runs:
readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
- name: Spilt Binary Symbols
shell: bash
run: |
objcopy --only-keep-debug ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
chmod 0644 ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
strip --strip-debug --remove-section=.comment --remove-section=.note ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
pushd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }} && objcopy --add-gnu-debuglink databend-query.debug databend-query && popd
# - name: Compress Binaries with UPX
# if: env.BUILD_PROFILE == 'debug'
# uses: crazy-max/ghaction-upx@v2
Expand Down
95 changes: 95 additions & 0 deletions .github/actions/pack_binaries/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
name: "Pack Binary"
description: "Pack releases binaries"
inputs:
target:
description: "Release target"
required: true
category:
description: "Release default/hdfs/udf/testsuite"
required: false
default: default
version:
description: "Release version"
required: true

runs:
using: "composite"
steps:
- name: Download artifact
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: ${{ inputs.target }}
category: ${{ inputs.category }}
path: distro/bin
artifacts: metactl,meta,query,query.debug
- name: Pack Binaries
id: pack_binaries
shell: bash
run: |
target=${{ inputs.target }}
version=${{ inputs.version }}
case ${{ inputs.category }} in
default)
pkg_name="databend-${version}-${target}"
;;
*)
pkg_name="databend-${{ inputs.category }}-${version}-${target}"
;;
esac
mkdir -p distro/{bin,configs,systemd,scripts}
cp ./scripts/distribution/systemd/databend-* distro/systemd/
cp ./scripts/distribution/configs/databend-* distro/configs/
cp ./scripts/distribution/release-readme.txt distro/readme.txt
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
- name: post sha256
uses: actions/upload-artifact@v4
with:
name: sha256sums-${{ inputs.category }}-${{ inputs.target }}
path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt
retention-days: 1
- name: post binaries
uses: actions/upload-artifact@v4
with:
name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
retention-days: 1
- name: Pack DBG Binaries
id: pack_dbg_binaries
shell: bash
run: |
target=${{ inputs.target }}
version=${{ inputs.version }}
case ${{ inputs.category }} in
default)
pkg_name="databend-${version}-${target}-dbg"
;;
*)
pkg_name="databend-${{ inputs.category }}-${version}-${target}-dbg"
;;
esac
mkdir -p distro/{bin,configs,systemd,scripts}
cp ./scripts/distribution/systemd/databend-* distro/systemd/
cp ./scripts/distribution/configs/databend-* distro/configs/
cp ./scripts/distribution/release-readme.txt distro/readme.txt
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
- name: post dbg sha256
uses: actions/upload-artifact@v4
with:
name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb
path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt
retention-days: 1
- name: post dbg binaries
uses: actions/upload-artifact@v4
with:
name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
retention-days: 1
94 changes: 94 additions & 0 deletions .github/actions/pack_distribution/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
name: "Pack Deb"
description: "Pack releases deb"
inputs:
arch:
description: "Release arch"
required: true
packager:
description: "Release default/hdfs/udf/testsuite"
required: false
default: default
version:
description: "Release version"
required: true

runs:
using: "composite"
steps:
- name: Install nfpm@latest
shell: bash
run: |
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
tar xf nfpm.tar.gz
sudo mv nfpm /usr/local/bin
sudo chmod a+x /usr/local/bin/nfpm
rm nfpm.tar.gz
- name: Get target
id: target
shell: bash
run: |
echo 'target=${{ inputs.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
- name: Download artifacts
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: ${{ steps.target.outputs.target }}
category: default
artifacts: metactl,meta,query,query.debug
path: distro/bin
- name: Build Packages
shell: bash
id: build_packages
run: |
export name="databend"
export version="${{ inputs.version }}"
export path="distro"
case "${{ inputs.arch }}" in
x86_64)
export arch="amd64"
;;
aarch64)
export arch="arm64"
;;
esac
deb_version=${version/-/.}
pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}.${{ inputs.packager }}"
mkdir -p distro/{bin,configs,systemd,scripts}
cp ./scripts/distribution/systemd/databend-* distro/systemd/
cp ./scripts/distribution/configs/databend-* distro/configs/
cp ./scripts/distribution/release-readme.txt distro/readme.txt
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
- name: Build dbg Packages
shell: bash
id: build_dbg_packages
run: |
export name="databend_dbg"
export version="${{ inputs.version }}"
export path="distro"
case "${{ inputs.arch }}" in
x86_64)
export arch="amd64"
;;
aarch64)
export arch="arm64"
;;
esac
deb_version=${version/-/.}
pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}_dbg.${{ inputs.packager }}"
nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm-dbg.yaml)
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
- name: Update release to github
shell: bash
env:
GH_TOKEN: ${{ github.token }}
# Reference: https://cli.github.com/manual/gh_release_upload
run: |
version="${{ inputs.version }}"
# name looks like: `databend_0.8.144~nightly_amd64.deb`
gh release upload ${version} ${{ steps.build_packages.outputs.pkg_name }} --clobber
gh release upload ${version} ${{ steps.build_dbg_packages.outputs.pkg_name }} --clobber
8 changes: 7 additions & 1 deletion .github/actions/publish_binary/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,19 @@ runs:
shell: bash
# Reference: https://cli.github.com/manual/gh_release_upload
run: |
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.* --clobber
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber
if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.* --clobber
fi
- name: Sync normal release to R2
shell: bash
if: inputs.category == 'default'
run: |
aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress
if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then
aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress
fi
gh api /repos/databendlabs/databend/tags > tags.json
aws s3 cp ./tags.json s3://repo/databend/tags.json
gh api /repos/databendlabs/databend/releases > releases.json
Expand Down
87 changes: 8 additions & 79 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -203,40 +203,12 @@ jobs:
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Download artifact
uses: ./.github/actions/artifact_download
- name: Pack Binaries
uses: ./.github/actions/pack_binaries
with:
sha: ${{ github.sha }}
version: ${{ needs.create_release.outputs.version }}
target: ${{ matrix.target }}
category: ${{ matrix.category }}
path: distro/bin
artifacts: metactl,meta,query
- name: Pack Binaries
run: |
target=${{ matrix.target }}
version=${{ needs.create_release.outputs.version }}
case ${{ matrix.category }} in
default)
pkg_name="databend-${version}-${target}"
;;
*)
pkg_name="databend-${{ matrix.category }}-${version}-${target}"
;;
esac
mkdir -p distro/{bin,configs,systemd,scripts}
cp ./scripts/distribution/systemd/databend-* distro/systemd/
cp ./scripts/distribution/configs/databend-* distro/configs/
cp ./scripts/distribution/release-readme.txt distro/readme.txt
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
- name: post sha256
uses: actions/upload-artifact@v4
with:
name: sha256sums-${{ matrix.category }}-${{ matrix.target }}
path: sha256-*.txt
retention-days: 1
- name: Publish Binaries
uses: ./.github/actions/publish_binary
env:
Expand Down Expand Up @@ -455,55 +427,12 @@ jobs:
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Install nfpm@latest
run: |
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
tar xf nfpm.tar.gz
sudo mv nfpm /usr/local/bin
sudo chmod a+x /usr/local/bin/nfpm
rm nfpm.tar.gz
- name: Get target
id: target
run: |
echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
- name: Download artifacts
uses: ./.github/actions/artifact_download
- name: Pack distribution
uses: ./.github/actions/pack_distribution
with:
sha: ${{ github.sha }}
target: ${{ steps.target.outputs.target }}
category: default
artifacts: metactl,meta,query
path: distro/bin
- name: Build Packages
id: build_packages
run: |
export name="databend"
export version="${{ needs.create_release.outputs.version }}"
export path="distro"
case "${{ matrix.arch }}" in
x86_64)
export arch="amd64"
;;
aarch64)
export arch="arm64"
;;
esac
mkdir -p distro/{bin,configs,systemd,scripts}
cp ./scripts/distribution/systemd/databend-* distro/systemd/
cp ./scripts/distribution/configs/databend-* distro/configs/
cp ./scripts/distribution/release-readme.txt distro/readme.txt
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
- name: Update release to github
shell: bash
env:
GH_TOKEN: ${{ github.token }}
# Reference: https://cli.github.com/manual/gh_release_upload
run: |
version="${{ needs.create_release.outputs.version }}"
# name looks like: `databend_0.8.144~nightly_amd64.deb`
gh release upload ${version} databend_*.${{ matrix.packager }} --clobber
version: ${{ needs.create_release.outputs.version }}
arch: ${{ matrix.arch }}
packager: ${{ matrix.packager }}

# bindings_python:
# if: inputs.stable
Expand Down
17 changes: 17 additions & 0 deletions scripts/distribution/nfpm-dbg.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
name: "${name}"
arch: "${arch}"
platform: "linux"
version: "${version}"
section: "database"
priority: "extra"
maintainer: "Databend Authors <[email protected]>"
description: |
Databend is a powerful cloud data warehouse. Built for elasticity and efficiency.
Free and open. Also available in the cloud: https://app.databend.com
vendor: "Datafuse Labs"
homepage: "https://databend.com"
license: "Apache-2.0"
contents:
# Binaries
- src: ${path}/bin/databend-query.debug
dst: /usr/bin/databend-query.debug
23 changes: 23 additions & 0 deletions scripts/setup/dev_setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,28 @@ function install_libtiff {
esac
}

function install_binutils {
PACKAGE_MANAGER=$1

echo "==> installing binutils..."

case "$PACKAGE_MANAGER" in
apt-get)
install_pkg binutils "$PACKAGE_MANAGER"
;;
yum | dnf)
install_pkg binutils "$PACKAGE_MANAGER"
;;
brew)
# skip
;;
*)
echo "Unable to install binutils with package manager: $PACKAGE_MANAGER"
exit 1
;;
esac
}

function install_rustup {
RUST_TOOLCHAIN=$1

Expand Down Expand Up @@ -624,6 +646,7 @@ if [[ "$INSTALL_BUILD_TOOLS" == "true" ]]; then
install_python3 "$PACKAGE_MANAGER"
install_sqlite3 "$PACKAGE_MANAGER"
install_libtiff "$PACKAGE_MANAGER"
install_binutils "$PACKAGE_MANAGER"

# Any call to cargo will make rustup install the correct toolchain
cargo version
Expand Down

0 comments on commit 7b98a13

Please sign in to comment.