Merge branch 'release-1.2' into protoc-update

Signed-off-by: Yury-Fridlyand <[email protected]>
valkey-io · Nov 20, 2024 · 2ea633d · 2ea633d
2 parents a985444 + ea5667b
commit 2ea633d
Show file tree

Hide file tree

Showing 52 changed files with 1,971 additions and 649 deletions.
diff --git a/.github/workflows/ort-sweeper.yml b/.github/workflows/ort-sweeper.yml
@@ -0,0 +1,57 @@
+name: ORT - Trigger periodic checks for relevant branches
+
+on:
+    schedule:
+        - cron: "0 0 * * *" # Runs daily at 00:00 UTC
+
+jobs:
+    trigger-ort-check:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Fetch relevant branches
+              id: get-branches
+              run: |
+                  # Get all branches matching 'release-*' and include 'main'
+                  branches=$(git ls-remote --heads origin | awk -F'/' '/refs\/heads\/release-/ {print $NF}')
+                  branches="main $branches"
+                  echo "::set-output name=branches::$branches"
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Trigger ORT Check workflows
+              uses: actions/github-script@v6
+              with:
+                  script: |
+                      const branches = "${{ steps.get-branches.outputs.branches }}".split(" ");
+                      const workflowFile = "ort.yml";
+
+                      const triggerWorkflow = async (branch) => {
+                        try {
+                          console.log(`Triggering workflow for branch: ${branch}`);
+                          await github.rest.actions.createWorkflowDispatch({
+                            owner: context.repo.owner,
+                            repo: context.repo.repo,
+                            workflow_id: workflowFile,
+                            ref: branch, // The branch where workflow_dispatch is triggered
+                            inputs: {
+                              branch_name: branch
+                            }
+                          });
+                          console.log(`Successfully triggered workflow for branch: ${branch}`);
+                        } catch (error) {
+                          core.setFailed(error.message);
+                        }
+                      };
+
+                      // Fire all workflow dispatch requests concurrently
+                      const promises = branches
+                        .filter(branch => branch) // Skip empty branches
+                        .map(branch => triggerWorkflow(branch));
+
+                      await Promise.allSettled(promises);
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml
@@ -1,57 +1,56 @@
 name: The OSS Review Toolkit (ORT)
 
 on:
-    schedule:
-        - cron: "0 0 * * *"
     pull_request:
-        paths:
-            - .github/workflows/ort.yml
-            - .github/workflows/run-ort-tools/action.yml
-            - utils/get_licenses_from_ort.py
+
     workflow_dispatch:
         inputs:
-            branch:
+            branch_name:
                 description: "The branch to run against the ORT tool"
                 required: true
-            version:
-                description: "The release version of GLIDE"
-                required: true
+                default: "main"
+
 jobs:
     run-ort:
-        if: github.repository_owner == 'valkey-io'
         name: Create attribution files
         runs-on: ubuntu-latest
+
+        # 1. For workflow_dispatch, always allow
+        # 2. For pull_request, run if branch is not autogenerated ort-diff-for- branches
+        if: >
+            github.event_name == 'workflow_dispatch' ||
+            (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'ort-diff-for-'))
         strategy:
             fail-fast: false
         env:
             PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON"
             NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE"
             RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST"
             JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA"
-        steps:
-            - name: Set the release version
-              shell: bash
-              run: |
-                  export version=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo '255.255.255'; else echo "$INPUT_VERSION"; fi`
-                  echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV
-              env:
-                  EVENT_NAME: ${{ github.event_name }}
-                  INPUT_VERSION: ${{ github.event.inputs.version }}
+            INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch_name }}
+            EVENT_NAME: ${{ github.event_name }}
+            HEAD_REF: ${{ github.head_ref }}
 
-            - name: Set the base branch
+        steps:
+            - name: Setup target branch and commit
               run: |
-                  export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi`
-                  echo "Base branch is: ${BASE_BRANCH}"
-                  echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV
-              env:
-                  EVENT_NAME: ${{ github.event_name }}
-                  INPUT_BRANCH: ${{ github.event.inputs.branch }}
+                  if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then
+                    echo "TARGET_BRANCH=$INPUT_TARGET_BRANCH" >> $GITHUB_ENV
+                  elif [ "$EVENT_NAME" == 'pull_request' ]; then
+                    echo "TARGET_BRANCH=$HEAD_REF" >> $GITHUB_ENV
+                  fi
 
-            - name: Checkout
+            - name: Checkout target branch
               uses: actions/checkout@v4
               with:
-                  submodules: "true"
-                  ref: ${{ env.BASE_BRANCH }}
+                  ref: ${{ env.TARGET_BRANCH }}
+                  repository: ${{ github.event.pull_request.head.repo.full_name }}
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  fetch-depth: 0 # Fetch all history for all branches and tags
+
+            - name: Setup target commit
+              run: |
+                  echo "TARGET_COMMIT=`git rev-parse HEAD`" >> $GITHUB_ENV
 
             - name: Set up JDK 11 for the ORT package
               uses: actions/setup-java@v4
@@ -81,7 +80,7 @@ jobs:
             - name: Install Rust toolchain
               uses: dtolnay/[email protected]
 
-            - name: Install ORT
+            - name: Build and install ORT
               if: steps.cache-ort.outputs.cache-hit != 'true'
               working-directory: ./ort/
               run: |
@@ -99,8 +98,7 @@ jobs:
                   EOF
                   cat ~/.ort/config/config.yml
 
-            ### NodeJS ###
-
+              ### NodeJS ###
             - name: Set up Node.js 16.x
               uses: actions/setup-node@v4
               with:
@@ -109,7 +107,7 @@ jobs:
             - name: Create package.json file for the Node wrapper
               uses: ./.github/workflows/node-create-package-file
               with:
-                  release_version: ${{ env.RELEASE_VERSION }}
+                  release_version: ${{ env.TARGET_COMMIT }}
                   os: "ubuntu-latest"
 
             - name: Fix Node base NPM package.json file for ORT
@@ -118,7 +116,7 @@ jobs:
                   # Remove the glide-rs dependency to avoid duplication 
                   sed -i '/ "glide-rs":/d' ../../package.json
                   export pkg_name=valkey-glide-base
-                  export package_version="${{ env.RELEASE_VERSION }}"
+                  export package_version="${{ env.TARGET_COMMIT }}"
                   export scope=`if [ "$NPM_SCOPE" != ''  ]; then echo "$NPM_SCOPE/"; fi`
                   mv package.json package.json.tmpl
                   envsubst < package.json.tmpl > "package.json"
@@ -129,8 +127,7 @@ jobs:
               with:
                   folder_path: "${{ github.workspace }}/node"
 
-            ### Python ###
-
+              ### Python ###
             - name: Set up Python 3.10
               uses: actions/setup-python@v5
               with:
@@ -147,15 +144,13 @@ jobs:
               with:
                   folder_path: "${{ github.workspace }}/python"
 
-            ### Rust ###
-
-            - name: Run ORT tools for Rust
+              ### Rust glide-core ###
+            - name: Run ORT tools for glide-core
               uses: ./.github/workflows/run-ort-tools
               with:
                   folder_path: "${{ github.workspace }}/glide-core"
 
-            ### Java ###
-
+              ### Java ###
             - name: Set up JDK 11
               uses: actions/setup-java@v4
               with:
@@ -167,21 +162,7 @@ jobs:
               with:
                   folder_path: "${{ github.workspace }}/java"
 
-            ### Process results ###
-
-            - name: Check for diff
-              run: |
-                  cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS
-                  cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS
-                  cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS
-                  cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS
-                  GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS`
-                  if [ -n "$GIT_DIFF" ]; then
-                  echo "FOUND_DIFF=true" >> $GITHUB_ENV
-                  else
-                  echo "FOUND_DIFF=false" >> $GITHUB_ENV
-                  fi
-
+              ### Get licenses ###
             - name: Retrieve licenses list
               working-directory: ./utils
               run: |
@@ -191,26 +172,7 @@ jobs:
                     echo EOF
                   } >> "$GITHUB_ENV"
 
-            ### Create PR ###
-
-            - name: Create pull request
-              if: ${{ env.FOUND_DIFF  == 'true' && github.event_name != 'pull_request' }}
-              run: |
-                  export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi`
-                  echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}"
-                  git config --global user.email "[email protected]"
-                  git config --global user.name "ort-bot"
-                  git checkout -b ${BRANCH_NAME}
-                  git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS
-                  git commit -m "Updated attribution files" -s
-                  git push --set-upstream origin ${BRANCH_NAME} -f
-                  title="Updated attribution files for ${BRANCH_NAME}"
-                  gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}'
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-                  EVENT_NAME: ${{ github.event_name }}
-                  INPUT_VERSION: ${{ github.event.inputs.version }}
-
+              ### Upload licenses ###
             - name: Get current date
               id: date
               run: |
@@ -234,3 +196,88 @@ jobs:
                   path: |
                       utils/skipped_package_list.txt
                   retention-days: 30
+
+            - name: Upload the unknown/unapproved package list
+              continue-on-error: true
+              uses: actions/upload-artifact@v4
+              with:
+                  name: unapproved-package-list-${{ steps.date.outputs.date }}
+                  path: |
+                      utils/unapproved_package_list.txt
+                  retention-days: 30
+
+              ### TODO: Fail if there are unapproved packages ###
+
+              ### Check for attributions diff ###
+            - name: Check for diff
+              run: |
+                  cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS
+                  cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS
+                  cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS
+                  cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS
+                  GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS`
+                  if [ -n "$GIT_DIFF" ]; then
+                    echo "FOUND_DIFF=true" >> $GITHUB_ENV
+                  else
+                    echo "FOUND_DIFF=false" >> $GITHUB_ENV
+                  fi
+
+              ### Create PR, Note a potential race on the source branch ###
+            - name: Create pull request
+              if: ${{ env.FOUND_DIFF  == 'true' && github.event_name != 'pull_request' }}
+              run: |
+                  export ORT_DIFF_BRANCH_NAME="ort-diff-for-$TARGET_BRANCH"
+                  echo "Creating pull request from branch $ORT_DIFF_BRANCH_NAME to branch $TARGET_BRANCH"
+                  git config --global user.email "[email protected]"
+                  git config --global user.name "ort-bot"
+                  git checkout -b ${ORT_DIFF_BRANCH_NAME}
+                  git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS
+                  git commit -m "Updated attribution files" -s
+                  git push --set-upstream origin ${ORT_DIFF_BRANCH_NAME} -f
+
+                  # Check if PR already exists
+                  existing_pr=$(gh pr list --base ${TARGET_BRANCH} --head ${ORT_DIFF_BRANCH_NAME} --json number --jq '.[0].number')
+
+                  if [ -z "$existing_pr" ]; then
+                    # Create a new PR if none exists
+                    title="Updated attribution files for commit ${TARGET_COMMIT}"
+                    gh pr create -B ${TARGET_BRANCH} -H ${ORT_DIFF_BRANCH_NAME} --title "${title}" --body "Created by Github action. ${{ env.LICENSES_LIST }}"
+                    echo "Pull request created successfully."
+                  else
+                    # Update the existing PR
+                    echo "Pull request #$existing_pr already exists. Updating branch."
+                    gh pr edit $existing_pr --title "Updated attribution files for commit ${TARGET_COMMIT}" --body "Created by Github action. ${{ env.LICENSES_LIST }}"
+                    echo "Pull request updated successfully."
+                  fi
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  INPUT_VERSION: ${{ github.event.inputs.version }}
+
+            ### Warn of outdated attributions for PR ###
+            - name: Warn of outdated attributions due to the PR
+              if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }}
+              run: |
+                  ATTRIBUTION_FILES=(
+                    "${{ env.PYTHON_ATTRIBUTIONS }}"
+                    "${{ env.NODE_ATTRIBUTIONS }}"
+                    "${{ env.RUST_ATTRIBUTIONS }}"
+                    "${{ env.JAVA_ATTRIBUTIONS }}"
+                  )
+
+                  MESSAGE="WARNING! The attribution files differ in this PR. Please ensure an updating PR is issued using a scheduled or manual run of this workflow!"
+
+                  # Echo the message to the console
+                  echo "$MESSAGE"
+
+                  # Emit a general warning in the action log
+                  echo "::warning::$MESSAGE"
+
+                  # Loop through the attribution files
+                  for FILE in "${ATTRIBUTION_FILES[@]}"; do
+                    if git diff --quiet "$FILE"; then
+                      continue
+                    else
+                      # Emit a warning associated with the changed file
+                      echo "::warning file=$FILE::WARNING! The attribution file '$FILE' differs in this PR."
+                    fi
+                  done
diff --git a/.github/workflows/redis-rs.yml b/.github/workflows/redis-rs.yml
@@ -65,6 +65,7 @@ jobs:
               working-directory: ./glide-core/redis-rs/redis
 
             - name: Test
+              # TODO remove the concurrency limit after we fix test flakyness.
               run: |
                   cargo test --release -- --test-threads=1 | tee ../test-results.xml
                   echo "### Tests passed :v:" >> $GITHUB_STEP_SUMMARY

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
 #### Changes
-* Node, Python: Adding support for replacing connection configured password ([#2651](https://github.com/valkey-io/valkey-glide/pull/2651))
+* Python: Client API for retrieving internal statistics ([#2707](https://github.com/valkey-io/valkey-glide/pull/2707))
+* Node, Python: Adding support for replacing connection configured password ([#2651](https://github.com/valkey-io/valkey-glide/pull/2651))([#2659](https://github.com/valkey-io/valkey-glide/pull/2659))
 * Node: Add FT._ALIASLIST command([#2652](https://github.com/valkey-io/valkey-glide/pull/2652))
 * Python: Python: `FT._ALIASLIST` command added([#2638](https://github.com/valkey-io/valkey-glide/pull/2638))
 * Node: alias commands added: FT.ALIASADD, FT.ALIADDEL, FT.ALIASUPDATE([#2596](https://github.com/valkey-io/valkey-glide/pull/2596))
@@ -83,6 +84,7 @@
 * Core: Add support for sending multi-slot JSON.MSET and JSON.MGET commands ([#2587]https://github.com/valkey-io/valkey-glide/pull/2587)
 * Node: Add `JSON.DEBUG` command ([#2572](https://github.com/valkey-io/valkey-glide/pull/2572))
 * Node: Add `JSON.NUMINCRBY` and `JSON.NUMMULTBY` command ([#2555](https://github.com/valkey-io/valkey-glide/pull/2555))
+* Core: Add support to Availability Zone Affinity read strategy ([#2539](https://github.com/valkey-io/valkey-glide/pull/2539))
 * Core: Fix list of readonly commands ([#2634](https://github.com/valkey-io/valkey-glide/pull/2634), [#2649](https://github.com/valkey-io/valkey-glide/pull/2649))
 * Core: Improve retry logic and update unmaintained dependencies for Rust lint CI ([#2673](https://github.com/valkey-io/valkey-glide/pull/2643))
 * Core: Release the read lock while creating connections in `refresh_connections` ([#2630](https://github.com/valkey-io/valkey-glide/issues/2630))

diff --git a/deny.toml b/deny.toml
@@ -57,7 +57,8 @@ allow = [
     "Unicode-DFS-2016",
     "ISC",
     "OpenSSL",
-    "MPL-2.0"
+    "MPL-2.0",
+    "Unicode-3.0"
 ]
 # The confidence threshold for detecting a license from license text.
 # The higher the value, the more closely the license text must be to the

diff --git a/glide-core/redis-rs/Cargo.toml b/glide-core/redis-rs/Cargo.toml
@@ -1,3 +1,9 @@
+# Dummy package so ORT tool will not fail on virtual workspace
+[package]
+name = "dummy-for-ort"
+version = "0.1.0"
+edition = "2021"
+
 [workspace]
-members = ["redis", "redis-test"]
+members = ["redis", "redis-test", "afl/parser"]
 resolver = "2"