From cd1fe2d76ff7ca1657fc2278d864558511f8e297 Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Tue, 26 Nov 2024 15:39:15 -0800 Subject: [PATCH 1/8] Improve aws creds workflow Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 15 ++++--- .../call-build-linux-arm-packages.yml | 39 ++++++++----------- .../call-build-linux-x86-packages.yml | 39 ++++++++----------- 3 files changed, 41 insertions(+), 52 deletions(-) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index 094d82de08..3f8ed95972 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -11,6 +11,7 @@ on: required: true permissions: + id-token: write contents: read jobs: @@ -67,11 +68,10 @@ jobs: version: ${{ needs.release-build-get-meta.outputs.version }} ref: ${{ inputs.version || github.ref_name }} build_matrix: ${{ needs.generate-build-matrix.outputs.x86_64-build-matrix }} + region: us-west-2 secrets: - token: ${{ secrets.GITHUB_TOKEN }} - bucket: ${{ secrets.AWS_S3_BUCKET }} - access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }} + bucket_name: ${{ secrets.AWS_S3_BUCKET }} + role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} release-build-linux-arm-packages: needs: @@ -82,8 +82,7 @@ jobs: version: ${{ needs.release-build-get-meta.outputs.version }} ref: ${{ inputs.version || github.ref_name }} build_matrix: ${{ needs.generate-build-matrix.outputs.arm64-build-matrix }} + region: us-west-2 secrets: - token: ${{ secrets.GITHUB_TOKEN }} - bucket: ${{ secrets.AWS_S3_BUCKET }} - access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }} + bucket_name: ${{ secrets.AWS_S3_BUCKET }} + role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} diff --git a/.github/workflows/call-build-linux-arm-packages.yml b/.github/workflows/call-build-linux-arm-packages.yml index 2a7bcc533f..65445a83c8 100644 --- a/.github/workflows/call-build-linux-arm-packages.yml +++ b/.github/workflows/call-build-linux-arm-packages.yml @@ -15,21 +15,20 @@ on: description: The build targets to produce as a JSON matrix. type: string required: true + region: + description: The AWS region to push packages into. + type: string + required: true secrets: - token: - description: The Github token or similar to authenticate with. + bucket_name: + description: The S3 bucket to push packages into. + required: true + role_to_assume: + description: The role to assume for the S3 bucket. required: true - bucket: - description: The name of the S3 bucket to push packages into. - required: false - access_key_id: - description: The S3 access key id for the bucket. - required: false - secret_access_key: - description: The S3 secret access key for the bucket. - required: false permissions: + id-token: write contents: read jobs: @@ -46,6 +45,12 @@ jobs: with: ref: ${{ inputs.version }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.region }} + role-to-assume: ${{ secrets.role_to_assume }} + - name: Make Valkey uses: uraimo/run-on-arch-action@v2 with: @@ -65,15 +70,5 @@ jobs: mkdir -p packages-files cp -rfv $TAR_FILE_NAME.tar* packages-files/ - - name: Install AWS cli. - run: | - sudo apt-get install -y awscli - - - name: Configure AWS credentials - run: | - aws configure set region us-west-2 - aws configure set aws_access_key_id ${{ secrets.access_key_id }} - aws configure set aws_secret_access_key ${{ secrets.secret_access_key }} - - name: Sync to S3 - run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/ + run: aws s3 sync packages-files s3://${{ secrets.bucket_name }}/releases/ diff --git a/.github/workflows/call-build-linux-x86-packages.yml b/.github/workflows/call-build-linux-x86-packages.yml index 9e438fa61a..a603c53c13 100644 --- a/.github/workflows/call-build-linux-x86-packages.yml +++ b/.github/workflows/call-build-linux-x86-packages.yml @@ -15,21 +15,20 @@ on: description: The build targets to produce as a JSON matrix. type: string required: true + region: + description: The AWS region to upload the packages to. + type: string + required: true secrets: - token: - description: The Github token or similar to authenticate with. + bucket_name: + description: The name of the S3 bucket to upload the packages to. + required: true + role_to_assume: + description: The role to assume for the S3 bucket. required: true - bucket: - description: The name of the S3 bucket to push packages into. - required: false - access_key_id: - description: The S3 access key id for the bucket. - required: false - secret_access_key: - description: The S3 secret access key for the bucket. - required: false permissions: + id-token: write contents: read jobs: @@ -46,6 +45,12 @@ jobs: with: ref: ${{ inputs.version }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.region }} + role-to-assume: ${{ secrets.role_to_assume }} + - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev libsystemd-dev @@ -63,15 +68,5 @@ jobs: mkdir -p packages-files cp -rfv $TAR_FILE_NAME.tar* packages-files/ - - name: Install AWS cli. - run: | - sudo apt-get install -y awscli - - - name: Configure AWS credentials - run: | - aws configure set region us-west-2 - aws configure set aws_access_key_id ${{ secrets.access_key_id }} - aws configure set aws_secret_access_key ${{ secrets.secret_access_key }} - - name: Sync to S3 - run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/ + run: aws s3 sync packages-files s3://${{ secrets.bucket_name }}/releases/ From 6d4c4577e4238baa8efdfedeb4ece060887172b7 Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Mon, 2 Dec 2024 13:25:16 -0800 Subject: [PATCH 2/8] AWS Test env CI Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 32 ++++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index 3f8ed95972..d0e3a775f7 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -3,7 +3,12 @@ name: Build Release Packages on: release: types: [published] - + pull_request: + paths: + - '.github/workflows/build-release-packages.yml' + - '.github/workflows/call-build-linux-arm-packages.yml' + - '.github/workflows/call-build-linux-x86_64-packages.yml' + - 'utils/releasetools/**' workflow_dispatch: inputs: version: @@ -21,8 +26,8 @@ jobs: runs-on: ubuntu-latest outputs: version: ${{ steps.get_version.outputs.VERSION }} + is_test: ${{ steps.check-env.outputs.IS_TEST }} steps: - - run: | echo "Version: ${{ inputs.version || github.ref_name }}" shell: bash @@ -33,8 +38,13 @@ jobs: - name: Get the version id: get_version run: | - VERSION="${INPUT_VERSION}" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + VERSION="unstable" + else + VERSION="${INPUT_VERSION}" + fi if [ -z "${VERSION}" ]; then + echo "Error: No version specified" exit 1 fi echo "VERSION=$VERSION" >> $GITHUB_OUTPUT @@ -44,6 +54,16 @@ jobs: # only ever be a tag INPUT_VERSION: ${{ inputs.version || github.ref_name }} + - name: Set bucket name + id: check-env + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "IS_TEST=true" >> $GITHUB_OUTPUT + else + echo "IS_TEST=false" >> $GITHUB_OUTPUT + fi + shell: bash + generate-build-matrix: name: Generating build matrix runs-on: ubuntu-latest @@ -57,7 +77,7 @@ jobs: - uses: ./.github/actions/generate-package-build-matrix id: set-matrix with: - ref: ${{ inputs.version || github.ref_name }} + ref: ${{ needs.release-build-get-meta.outputs.version }} release-build-linux-x86-packages: needs: @@ -70,7 +90,7 @@ jobs: build_matrix: ${{ needs.generate-build-matrix.outputs.x86_64-build-matrix }} region: us-west-2 secrets: - bucket_name: ${{ secrets.AWS_S3_BUCKET }} + bucket_name: ${{ needs.release-build-get-meta.outputs.is_test == 'true' && secrets.AWS_TEST_BUCKET || secrets.AWS_S3_BUCKET }} role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} release-build-linux-arm-packages: @@ -84,5 +104,5 @@ jobs: build_matrix: ${{ needs.generate-build-matrix.outputs.arm64-build-matrix }} region: us-west-2 secrets: - bucket_name: ${{ secrets.AWS_S3_BUCKET }} + bucket_name: ${{ needs.release-build-get-meta.outputs.is_test == 'true' && secrets.AWS_TEST_BUCKET || secrets.AWS_S3_BUCKET }} role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} From 1d97e1cedbd3543ffdd66e098f9f3b3cd5923f77 Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Fri, 6 Dec 2024 14:34:40 -0800 Subject: [PATCH 3/8] Change version to the target branch Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index d0e3a775f7..37204f8c8b 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -39,7 +39,7 @@ jobs: id: get_version run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then - VERSION="unstable" + VERSION=${{ github.base_ref }} else VERSION="${INPUT_VERSION}" fi From cb55543ffff2ec0d41ee2e81e865589b7a3be402 Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Tue, 10 Dec 2024 15:31:27 -0800 Subject: [PATCH 4/8] Change steps name and id-name Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index 37204f8c8b..55676ea775 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest outputs: version: ${{ steps.get_version.outputs.VERSION }} - is_test: ${{ steps.check-env.outputs.IS_TEST }} + is_test: ${{ steps.check-if-testing.outputs.IS_TEST }} steps: - run: | echo "Version: ${{ inputs.version || github.ref_name }}" @@ -54,8 +54,8 @@ jobs: # only ever be a tag INPUT_VERSION: ${{ inputs.version || github.ref_name }} - - name: Set bucket name - id: check-env + - name: Check if we are testing + id: check-if-testing run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "IS_TEST=true" >> $GITHUB_OUTPUT From 96a61a34b888a0632e236dfcfd059c5f3915bbc9 Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Wed, 11 Dec 2024 13:59:51 -0800 Subject: [PATCH 5/8] resolve pull-request to push Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index 55676ea775..a8524bd3ee 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -3,7 +3,7 @@ name: Build Release Packages on: release: types: [published] - pull_request: + push: paths: - '.github/workflows/build-release-packages.yml' - '.github/workflows/call-build-linux-arm-packages.yml' @@ -38,7 +38,7 @@ jobs: - name: Get the version id: get_version run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then + if [[ "${{ github.event_name }}" == "push" ]]; then VERSION=${{ github.base_ref }} else VERSION="${INPUT_VERSION}" @@ -57,7 +57,7 @@ jobs: - name: Check if we are testing id: check-if-testing run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then + if [[ "${{ github.event_name }}" == "push" ]]; then echo "IS_TEST=true" >> $GITHUB_OUTPUT else echo "IS_TEST=false" >> $GITHUB_OUTPUT From facaf6895cfc3c436c728324b7d63d9ba6a46348 Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Wed, 11 Dec 2024 14:03:41 -0800 Subject: [PATCH 6/8] change target branch Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index a8524bd3ee..1eca19b570 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -5,9 +5,6 @@ on: types: [published] push: paths: - - '.github/workflows/build-release-packages.yml' - - '.github/workflows/call-build-linux-arm-packages.yml' - - '.github/workflows/call-build-linux-x86_64-packages.yml' - 'utils/releasetools/**' workflow_dispatch: inputs: @@ -39,7 +36,7 @@ jobs: id: get_version run: | if [[ "${{ github.event_name }}" == "push" ]]; then - VERSION=${{ github.base_ref }} + VERSION=${{ github.ref_name }} else VERSION="${INPUT_VERSION}" fi From 03cc5c14b9adec2b6d9d11e5ba14ea41ddee623a Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Wed, 11 Dec 2024 14:57:08 -0800 Subject: [PATCH 7/8] paths Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index 1eca19b570..c54e2df8ed 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -5,6 +5,9 @@ on: types: [published] push: paths: + - '.github/workflows/build-release-packages.yml' + - '.github/workflows/call-build-linux-arm-packages.yml' + - '.github/workflows/call-build-linux-x86_64-packages.yml' - 'utils/releasetools/**' workflow_dispatch: inputs: From f2ea7cba86f84509ae5231849898cdb2026280f5 Mon Sep 17 00:00:00 2001 From: vudiep411 Date: Wed, 11 Dec 2024 15:09:06 -0800 Subject: [PATCH 8/8] paths Signed-off-by: vudiep411 --- .github/workflows/build-release-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index c54e2df8ed..44e012d658 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -8,7 +8,7 @@ on: - '.github/workflows/build-release-packages.yml' - '.github/workflows/call-build-linux-arm-packages.yml' - '.github/workflows/call-build-linux-x86_64-packages.yml' - - 'utils/releasetools/**' + - 'utils/releasetools/build-config.json' workflow_dispatch: inputs: version: