From 985b4e6928a66b9cdf84755a240c94d13a3fd99a Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Mon, 15 Apr 2024 18:20:53 -0700 Subject: [PATCH 1/5] Fix redis installation on macos and amazonlinux (#199) * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand * Experiment Signed-off-by: Yury-Fridlyand --------- Signed-off-by: Yury-Fridlyand --- .github/workflows/csharp.yml | 14 ++++---- .github/workflows/go.yml | 8 +---- .github/workflows/install-redis/action.yml | 16 ++++----- .../install-shared-dependencies/action.yml | 4 +-- .github/workflows/java.yml | 16 +++------ .github/workflows/node.yml | 34 +++++++++++++++---- .github/workflows/python.yml | 30 ++++++++++++---- 7 files changed, 71 insertions(+), 51 deletions(-) diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index b238f3ac18..a38c259b15 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -32,7 +32,7 @@ concurrency: jobs: run-tests: - timeout-minutes: 15 + timeout-minutes: 25 strategy: fail-fast: false matrix: @@ -52,13 +52,6 @@ jobs: with: submodules: recursive - - name: Install redis - # TODO: make this step macos compatible: https://github.com/aws/glide-for-redis/issues/781 - if: ${{ matrix.os == 'ubuntu-latest' }} - uses: ./.github/workflows/install-redis - with: - redis-version: ${{ matrix.redis }} - - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: @@ -66,6 +59,11 @@ jobs: target: ${{ matrix.os == 'ubuntu-latest' && 'x86_64-unknown-linux-gnu' || 'x86_64-apple-darwin' }} github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install redis + uses: ./.github/workflows/install-redis + with: + redis-version: ${{ matrix.redis }} + - name: Set up dotnet ${{ matrix.dotnet }} uses: actions/setup-dotnet@v4 with: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 805c46ba6c..65acd999b6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -58,7 +58,6 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install redis - # TODO: make this step macos compatible: https://github.com/aws/glide-for-redis/issues/781 if: ${{ matrix.os == 'ubuntu-latest' }} uses: ./.github/workflows/install-redis with: @@ -85,7 +84,7 @@ jobs: run: make test build-amazonlinux-latest: - if: github.repository_owner == 'aws' +# if: github.repository_owner == 'aws' strategy: # Run all jobs fail-fast: false @@ -120,11 +119,6 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Create a symbolic Link for redis6 binaries - run: | - ln -s /usr/bin/redis6-server /usr/bin/redis-server - ln -s /usr/bin/redis6-cli /usr/bin/redis-cli - - name: Install Go run: | yum -y install wget diff --git a/.github/workflows/install-redis/action.yml b/.github/workflows/install-redis/action.yml index f4a7491d04..179aceace9 100644 --- a/.github/workflows/install-redis/action.yml +++ b/.github/workflows/install-redis/action.yml @@ -28,20 +28,16 @@ runs: ~/redis-binaries/${{ inputs.redis-version }}/redis-server key: ${{ runner.os }}-${{ inputs.redis-version }}-install-redis - - name: Install redis + - name: Build redis shell: bash if: steps.cache-redis.outputs.cache-hit != 'true' run: | - sudo apt-get update wget https://github.com/redis/redis/archive/${{ inputs.redis-version }}.tar.gz; tar -xzvf ${{ inputs.redis-version }}.tar.gz; - pushd redis-${{ inputs.redis-version }} && BUILD_TLS=yes make && sudo mv src/redis-server src/redis-cli ~/redis-binaries/${{ inputs.redis-version }} && popd; - - - name: Remove the source package - shell: bash - if: steps.cache-redis.outputs.cache-hit != 'true' - run: sudo rm -r redis-${{ inputs.redis-version }} + mkdir -p ~/redis-binaries/${{ inputs.redis-version }} + pushd redis-${{ inputs.redis-version }} && BUILD_TLS=yes make && mv src/redis-server src/redis-cli ~/redis-binaries/${{ inputs.redis-version }} && popd; - - name: Copy executable to place + - name: Install shell: bash - run: sudo cp ~/redis-binaries/${{ inputs.redis-version }}/redis-server ~/redis-binaries/${{ inputs.redis-version }}/redis-cli /usr/bin/ + run: | + echo $HOME/redis-binaries/${{ inputs.redis-version }} >> $GITHUB_PATH diff --git a/.github/workflows/install-shared-dependencies/action.yml b/.github/workflows/install-shared-dependencies/action.yml index 42747ac0be..574acdac38 100644 --- a/.github/workflows/install-shared-dependencies/action.yml +++ b/.github/workflows/install-shared-dependencies/action.yml @@ -33,7 +33,7 @@ runs: run: | brew update brew upgrade || true - brew install git gcc pkgconfig openssl redis coreutils + brew install git gcc pkgconfig openssl coreutils - name: Install software dependencies for Ubuntu shell: bash @@ -46,7 +46,7 @@ runs: shell: bash if: "${{ inputs.os == 'amazon-linux' }}" run: | - yum install -y gcc pkgconfig openssl openssl-devel which curl redis6 gettext --allowerasing + yum install -y gcc pkgconfig openssl openssl-devel which curl gettext wget jemalloc-devel lua-devel make gcc tar gzip --allowerasing - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 1de3352313..8175cf073a 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -65,8 +65,6 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install redis - # TODO: make this step macos compatible: https://github.com/aws/glide-for-redis/issues/781 - if: ${{ matrix.os == 'ubuntu-latest' }} uses: ./.github/workflows/install-redis with: redis-version: ${{ matrix.redis }} @@ -96,7 +94,7 @@ jobs: benchmarks/results/** build-amazonlinux-latest: - if: github.repository_owner == 'aws' +# if: github.repository_owner == 'aws' strategy: # Run all jobs fail-fast: false @@ -131,19 +129,15 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Create a symbolic Link for redis6 binaries - run: | - ln -s /usr/bin/redis6-server /usr/bin/redis-server - ln -s /usr/bin/redis6-cli /usr/bin/redis-cli + - name: Install redis + uses: ./.github/workflows/install-redis + with: + redis-version: 6.2.14 - name: Install Java run: | yum install -y java-${{ matrix.java }} - - name: Build rust part - working-directory: java - run: cargo build --release - - name: Build java part working-directory: java run: ./gradlew --continue build diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 4248e4fc81..50b8b084db 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -125,6 +125,18 @@ jobs: run: | npm i -g npm@8 + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: "macos-latest" + target: "x86_64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install redis + uses: ./.github/workflows/install-redis + with: + redis-version: 6.2.14 + - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: @@ -161,6 +173,18 @@ jobs: run: | yum install -y nodejs + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: "amazon-linux" + target: "x86_64-unknown-linux-gnu" + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install redis + uses: ./.github/workflows/install-redis + with: + redis-version: 6.2.14 + - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: @@ -168,12 +192,8 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Create a symbolic Link for redis6 binaries - working-directory: ./node - run: | - ln -s /usr/bin/redis6-server /usr/bin/redis-server - ln -s /usr/bin/redis6-cli /usr/bin/redis-cli - - name: Test compatibility - run: npm test -- -t "set and get flow works" working-directory: ./node + run: | + npm install --save-dev jest + npm test -- -t "set and get flow works" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a61b257eb9..4f48e5a137 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -138,6 +138,18 @@ jobs: - name: Set up Homebrew uses: Homebrew/actions/setup-homebrew@master + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: "macos-latest" + target: "x86_64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install redis + uses: ./.github/workflows/install-redis + with: + redis-version: 6.2.14 + - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper with: @@ -175,6 +187,18 @@ jobs: run: | yum install -y python3 + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: "amazon-linux" + target: "x86_64-unknown-linux-gnu" + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install redis + uses: ./.github/workflows/install-redis + with: + redis-version: 6.2.14 + - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper with: @@ -182,12 +206,6 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Create a symbolic Link for redis6 binaries - working-directory: ./python - run: | - ln -s /usr/bin/redis6-server /usr/bin/redis-server - ln -s /usr/bin/redis6-cli /usr/bin/redis-cli - - name: Test compatibility with pytest working-directory: ./python run: | From 5fb231b80ed9d7f545d3e1575da76d30c9d82da6 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 17 Apr 2024 08:34:30 -0700 Subject: [PATCH 2/5] Revert some changes Signed-off-by: Yury-Fridlyand --- .github/workflows/go.yml | 3 +-- .github/workflows/install-redis/action.yml | 8 +++++--- .github/workflows/java.yml | 2 +- .github/workflows/node.yml | 13 +++---------- .github/workflows/python.yml | 11 ++--------- 5 files changed, 12 insertions(+), 25 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 65acd999b6..5571a0800d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -58,7 +58,6 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install redis - if: ${{ matrix.os == 'ubuntu-latest' }} uses: ./.github/workflows/install-redis with: redis-version: ${{ matrix.redis }} @@ -84,7 +83,7 @@ jobs: run: make test build-amazonlinux-latest: -# if: github.repository_owner == 'aws' + if: github.repository_owner == 'aws' strategy: # Run all jobs fail-fast: false diff --git a/.github/workflows/install-redis/action.yml b/.github/workflows/install-redis/action.yml index 179aceace9..19b81eaffc 100644 --- a/.github/workflows/install-redis/action.yml +++ b/.github/workflows/install-redis/action.yml @@ -34,10 +34,12 @@ runs: run: | wget https://github.com/redis/redis/archive/${{ inputs.redis-version }}.tar.gz; tar -xzvf ${{ inputs.redis-version }}.tar.gz; - mkdir -p ~/redis-binaries/${{ inputs.redis-version }} - pushd redis-${{ inputs.redis-version }} && BUILD_TLS=yes make && mv src/redis-server src/redis-cli ~/redis-binaries/${{ inputs.redis-version }} && popd; + mkdir -p $HOME/redis-binaries/${{ inputs.redis-version }} + pushd redis-${{ inputs.redis-version }} + BUILD_TLS=yes make + mv src/redis-server src/redis-cli $HOME/redis-binaries/${{ inputs.redis-version }} - - name: Install + - name: Install redis shell: bash run: | echo $HOME/redis-binaries/${{ inputs.redis-version }} >> $GITHUB_PATH diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 3b4d371109..e7cadee3d3 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -100,7 +100,7 @@ jobs: benchmarks/results/** build-amazonlinux-latest: -# if: github.repository_owner == 'aws' + if: github.repository_owner == 'aws' strategy: # Run all jobs fail-fast: false diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 50b8b084db..c95344b990 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -125,10 +125,11 @@ jobs: run: | npm i -g npm@8 - - name: Install shared software dependencies - uses: ./.github/workflows/install-shared-dependencies + - name: Build Node wrapper + uses: ./.github/workflows/build-node-wrapper with: os: "macos-latest" + named_os: "darwin" target: "x86_64-apple-darwin" github-token: ${{ secrets.GITHUB_TOKEN }} @@ -137,14 +138,6 @@ jobs: with: redis-version: 6.2.14 - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: "macos-latest" - named_os: "darwin" - target: "x86_64-apple-darwin" - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Test compatibility run: npm test -- -t "set and get flow works" working-directory: ./node diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4f48e5a137..02ede83b06 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -138,8 +138,8 @@ jobs: - name: Set up Homebrew uses: Homebrew/actions/setup-homebrew@master - - name: Install shared software dependencies - uses: ./.github/workflows/install-shared-dependencies + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper with: os: "macos-latest" target: "x86_64-apple-darwin" @@ -150,13 +150,6 @@ jobs: with: redis-version: 6.2.14 - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: "macos-latest" - target: "x86_64-apple-darwin" - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Test compatibility with pytest working-directory: ./python run: | From d12df6fb4847edff1a94299fcdcd8d5592bde9bc Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 17 Apr 2024 08:53:53 -0700 Subject: [PATCH 3/5] Try to fix CI Signed-off-by: Yury-Fridlyand --- .github/workflows/install-redis/action.yml | 2 +- .github/workflows/node.yml | 4 +++- .github/workflows/python.yml | 11 ++--------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/install-redis/action.yml b/.github/workflows/install-redis/action.yml index 19b81eaffc..76dddfe2ad 100644 --- a/.github/workflows/install-redis/action.yml +++ b/.github/workflows/install-redis/action.yml @@ -35,7 +35,7 @@ runs: wget https://github.com/redis/redis/archive/${{ inputs.redis-version }}.tar.gz; tar -xzvf ${{ inputs.redis-version }}.tar.gz; mkdir -p $HOME/redis-binaries/${{ inputs.redis-version }} - pushd redis-${{ inputs.redis-version }} + cd redis-${{ inputs.redis-version }} BUILD_TLS=yes make mv src/redis-server src/redis-cli $HOME/redis-binaries/${{ inputs.redis-version }} diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index c95344b990..6440f9b966 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -139,8 +139,10 @@ jobs: redis-version: 6.2.14 - name: Test compatibility - run: npm test -- -t "set and get flow works" working-directory: ./node + run: | + npm install --save-dev jest + npm test -- -t "set and get flow works" build-amazonlinux-latest: runs-on: ubuntu-latest diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 02ede83b06..a8c3221b4d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -180,8 +180,8 @@ jobs: run: | yum install -y python3 - - name: Install shared software dependencies - uses: ./.github/workflows/install-shared-dependencies + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper with: os: "amazon-linux" target: "x86_64-unknown-linux-gnu" @@ -192,13 +192,6 @@ jobs: with: redis-version: 6.2.14 - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Test compatibility with pytest working-directory: ./python run: | From e9553cf9f6915ef2fe0c8fe7950501ed76872d0a Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 17 Apr 2024 09:22:21 -0700 Subject: [PATCH 4/5] Experiment Signed-off-by: Yury-Fridlyand --- .github/workflows/node.yml | 25 ++----------------------- .github/workflows/python.yml | 10 ---------- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 6440f9b966..d107dd9209 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -133,16 +133,9 @@ jobs: target: "x86_64-apple-darwin" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install redis - uses: ./.github/workflows/install-redis - with: - redis-version: 6.2.14 - - name: Test compatibility + run: npm test -- -t "set and get flow works" working-directory: ./node - run: | - npm install --save-dev jest - npm test -- -t "set and get flow works" build-amazonlinux-latest: runs-on: ubuntu-latest @@ -168,18 +161,6 @@ jobs: run: | yum install -y nodejs - - name: Install shared software dependencies - uses: ./.github/workflows/install-shared-dependencies - with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install redis - uses: ./.github/workflows/install-redis - with: - redis-version: 6.2.14 - - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: @@ -188,7 +169,5 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Test compatibility + run: npm test -- -t "set and get flow works" working-directory: ./node - run: | - npm install --save-dev jest - npm test -- -t "set and get flow works" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a8c3221b4d..e7dd71adc4 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -145,11 +145,6 @@ jobs: target: "x86_64-apple-darwin" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install redis - uses: ./.github/workflows/install-redis - with: - redis-version: 6.2.14 - - name: Test compatibility with pytest working-directory: ./python run: | @@ -187,11 +182,6 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install redis - uses: ./.github/workflows/install-redis - with: - redis-version: 6.2.14 - - name: Test compatibility with pytest working-directory: ./python run: | From af99feb91125d483054e89318cb32845e09e880d Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 19 Apr 2024 16:20:12 -0700 Subject: [PATCH 5/5] Experiment Signed-off-by: Yury-Fridlyand --- .github/workflows/node.yml | 6 ++++++ .github/workflows/python.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index d107dd9209..4248e4fc81 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -168,6 +168,12 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Create a symbolic Link for redis6 binaries + working-directory: ./node + run: | + ln -s /usr/bin/redis6-server /usr/bin/redis-server + ln -s /usr/bin/redis6-cli /usr/bin/redis-cli + - name: Test compatibility run: npm test -- -t "set and get flow works" working-directory: ./node diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index e7dd71adc4..a61b257eb9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -182,6 +182,12 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Create a symbolic Link for redis6 binaries + working-directory: ./python + run: | + ln -s /usr/bin/redis6-server /usr/bin/redis-server + ln -s /usr/bin/redis6-cli /usr/bin/redis-cli + - name: Test compatibility with pytest working-directory: ./python run: |