Node: Add server-modules to npm release (#2754)

Signed-off-by: Andrew Carbonetto <[email protected]>
Signed-off-by: avifenesh <[email protected]>

.github/DEVELOPER.md

@@ -133,7 +133,7 @@ We use dynamic matrices for our CI/CD workflows, which are created using the `cr
 4. It generates three matrices:
     - Engine matrix: Defines the types and versions of the engine to test against, for example Valkey 7.2.5.
     - Host matrix: Defines the host platforms to run the tests on, for example Ubuntu on ARM64.
-    - Language-version matrix: Defines the supported versions of languages, for example python 3.8.
+    - Language-version matrix: Defines the supported versions of languages, for example python 3.9.
 
 #### Outputs
 

.github/json_matrices/supported-languages-versions.json

@@ -6,8 +6,8 @@
     },
     {
         "language": "python",
-        "versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"],
-        "always-run-versions": ["3.8", "3.13"]
+        "versions": ["3.9", "3.10", "3.11", "3.12", "3.13"],
+        "always-run-versions": ["3.9", "3.13"]
     },
     {
         "language": "node",

.github/workflows/pypi-cd.yml

@@ -79,8 +79,6 @@ jobs:
 
             - name: Checkout
               uses: actions/checkout@v4
-              with:
-                  submodules: "true"
 
             - name: Set the release version
               shell: bash
@@ -116,7 +114,14 @@ jobs:
               if: ${{ !contains(matrix.build.RUNNER, 'self-hosted') }}
               uses: actions/setup-python@v5
               with:
-                  python-version: "3.12"
+                  python-version: "3.13"
+
+            - name: Setup Python for self-hosted Ubuntu runners
+              if: contains(matrix.build.RUNNER, 'self-hosted')
+              run: |
+                  sudo apt update -y
+                  sudo apt upgrade -y
+                  sudo apt install python3 python3-venv python3-pip -y
 
             - name: Update package version in config.toml
               uses: ./.github/workflows/update-glide-version
@@ -140,44 +145,59 @@ jobs:
                   # Log the edited .gitignore file
                   cat .gitignore
 
-            - name: Build Python wheels (linux)
-              if: startsWith(matrix.build.NAMED_OS, 'linux')
+            - name: Set python versions to build
+              run: |
+                  if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+                    echo "PYTHON_VERSIONS=python3.13" >> $GITHUB_ENV
+                  else
+                    echo "PYTHON_VERSIONS='python3.9 python3.10 python3.11 python3.12 python3.13'" >> $GITHUB_ENV
+                  fi
+
+            - name: Build Python wheels (linux x86_64)
+              if: matrix.build.TARGET == 'x86_64-unknown-linux-gnu'
               uses: PyO3/maturin-action@v1
               with:
-                  working-directory: ./python
-                  target: ${{ matrix.build.TARGET }}
-                  args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }}
-                  manylinux: auto
-                  container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }}
-                  before-script-linux: |
-                      # Install protobuf compiler
-                      if [[ $(`which apt`) != '' ]]
-                      then
-                        echo "installing unzip and curl"
-                        apt install unzip curl -y
-                      fi
-                      PB_REL="https://github.com/protocolbuffers/protobuf/releases"
-                      ARCH=`uname -p`
-                      if [[ $ARCH == 'x86_64' ]]; then 
-                        PROTOC_ARCH="x86_64"
-                      elif [[ $ARCH == 'aarch64' ]]; then 
-                        PROTOC_ARCH="aarch_64"
-                      else 
-                        echo "Running on unsupported architecture: $ARCH. Expected one of: ['x86_64', 'aarch64']"
-                        exit 1 
-                      fi
-                      curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-${PROTOC_ARCH}.zip
-                      unzip protoc-3.20.3-linux-${PROTOC_ARCH}.zip -d $HOME/.local
-                      export PATH="$PATH:$HOME/.local/bin"
+                working-directory: ./python
+                target: ${{ matrix.build.TARGET }}
+                args: --release --strip --out wheels -i ${PYTHON_VERSIONS}
+                manylinux: auto
+                container: ${{ matrix.build.CONTAINER }}
+                before-script-linux: |
+                  # Install protobuf compiler
+                  apt update && apt install unzip curl -y
+                  PB_REL="https://github.com/protocolbuffers/protobuf/releases"
+                  curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-x86_64.zip
+                  unzip protoc-3.20.3-linux-x86_64.zip -d $HOME/.local
+                  export PATH="$PATH:$HOME/.local/bin"
+
+            - name: Build Python wheels (linux aarch64)
+              if: matrix.build.TARGET == 'aarch64-unknown-linux-gnu'
+              uses: PyO3/maturin-action@v1
+              with:
+                working-directory: ./python
+                target: ${{ matrix.build.TARGET }}
+                args: --release --strip --out wheels -i ${PYTHON_VERSIONS}
+                manylinux: "2014"
+                container: ${{ matrix.build.CONTAINER }}
+                before-script-linux: |
+                  # Install protobuf compiler
+                  apt update && apt install unzip curl -y
+                  PB_REL="https://github.com/protocolbuffers/protobuf/releases"
+                  curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-aarch_64.zip
+                  unzip protoc-3.20.3-linux-aarch_64.zip -d $HOME/.local
+                  export PATH="$PATH:$HOME/.local/bin"
+              env:
+                CFLAGS: "-D__ARM_ARCH=8"
+                RUSTFLAGS: "-C target-feature=+neon"
 
             - name: Build Python wheels (macos)
               if: startsWith(matrix.build.NAMED_OS, 'darwin')
               uses: PyO3/maturin-action@v1
               with:
-                  maturin-version: latest
+                  maturin-version: 0.14.14                 
                   working-directory: ./python
                   target: ${{ matrix.build.TARGET }}
-                  args: --release --strip --out wheels -i  ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }}
+                  args: --release --strip --out wheels -i ${PYTHON_VERSIONS}
 
             - name: Upload Python wheels
               if: github.event_name != 'pull_request'
@@ -217,6 +237,10 @@ jobs:
             matrix:
                 build: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }}
         steps:
+            - name: Setup self-hosted runner access
+              if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }}
+              run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide
+
             - name: checkout
               uses: actions/checkout@v4
 
@@ -228,20 +252,20 @@ jobs:
             - name: Install engine
               uses: ./.github/workflows/install-engine
               with:
-                  version: "8.0"
+                  engine-version: "8.0"
+                  target: ${{ matrix.build.TARGET }}
 
             - name: Check if RC and set a distribution tag for the package
               shell: bash
               run: |
                   if [[ "${GITHUB_REF:11}" == *"rc"* ]]
                   then
                     echo "This is a release candidate"
-                    export pip_pre="--pre"
+                    echo "DISTRIBUTION_TAG=--pre" >> $GITHUB_ENV
                   else
                     echo "This is a stable release"
-                    export pip_pre=""
+                    echo "DISTRIBUTION_TAG=" >> $GITHUB_ENV
                   fi
-                  echo "PIP_PRE=${pip_pre}" >> $GITHUB_ENV
 
             - name: Run the tests
               shell: bash
@@ -250,7 +274,7 @@ jobs:
                   python -m venv venv
                   source venv/bin/activate
                   pip install -U pip
-                  pip install ${PIP_PRE} valkey-glide
+                  pip install ${DISTRIBUTION_TAG} valkey-glide
                   python rc_test.py
 
             # Reset the repository to make sure we get the clean checkout of the action later in other actions.

node/npm/glide/package.json

@@ -10,8 +10,8 @@
         "lint": "eslint .",
         "lint:fix": "eslint . --fix",
         "clean": "rm -rf build-ts/",
-        "copy-declaration-files": "cp ../../build-ts/*.d.ts build-ts/ && cp ../../build-ts/src/*.d.ts build-ts/src/",
-        "build": "tsc && mkdir -p build-ts/src && npm run copy-declaration-files"
+        "copy-declaration-files": "cp ../../build-ts/*.d.ts build-ts/ && cp ../../build-ts/src/*.d.ts build-ts/src/ && cp ../../build-ts/src/server-modules/*.d.ts build-ts/src/server-modules/",
+        "build": "tsc && mkdir -p build-ts/src && mkdir -p build-ts/src/server-modules && npm run copy-declaration-files"
     },
     "files": [
         "/build-ts"

python/Pipfile

@@ -8,4 +8,4 @@ name = "pypi"
 [dev-packages]
 
 [requires]
-python_version = "3.8"
+python_version = "3.9"

python/README.md

@@ -25,11 +25,11 @@ macOS:
 
 | Python Version |
 |----------------|
-| 3.8            |
 | 3.9            |
 | 3.10           |
 | 3.11           |
 | 3.12           |
+| 3.13           |
 
 ## Installation and Setup
 

python/pyproject.toml

@@ -1,14 +1,14 @@
 [build-system]
-requires = ["maturin>=0.13,<0.14"]
+requires = ["maturin>=1.0.0"]
 build-backend = "maturin"
 
 [project]
 name = "valkey-glide"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
     "async-timeout>=4.0.2; python_version < '3.11'",
     "typing-extensions>=4.8.0; python_version < '3.11'",
-    "protobuf>=3.20"
+    "protobuf>=3.20",
 ]
 classifiers = [
     "Topic :: Database",
@@ -30,7 +30,7 @@ max-line-length = 127
 extend-ignore = ['E203']
 
 [tool.black]
-target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
+target-version = ['py39', 'py310', 'py311', 'py312', 'py313']
 
 [tool.mypy]
-exclude = [ 'submodules', 'utils/release-candidate-testing' ]
+exclude = ['submodules', 'utils/release-candidate-testing']

python/python/glide/async_commands/standalone_commands.py

@@ -2,19 +2,17 @@
 
 from __future__ import annotations
 
-from typing import Any, Dict, List, Mapping, Optional, Set, Union, cast
+from typing import Dict, List, Mapping, Optional, Union, cast
 
-from glide.async_commands.command_args import Limit, ObjectType, OrderBy
+from glide.async_commands.command_args import ObjectType
 from glide.async_commands.core import (
     CoreCommands,
     FlushMode,
     FunctionRestorePolicy,
     InfoSection,
-    _build_sort_args,
 )
 from glide.async_commands.transaction import Transaction
 from glide.constants import (
-    OK,
     TOK,
     TEncodable,
     TFunctionListResponse,
@@ -23,7 +21,7 @@
 )
 from glide.protobuf.command_request_pb2 import RequestType
 
-from ..glide import ClusterScanCursor, Script
+from ..glide import Script
 
 
 class StandaloneCommands(CoreCommands):

python/python/tests/conftest.py

@@ -12,7 +12,7 @@
     ReadFrom,
     ServerCredentials,
 )
-from glide.exceptions import ClosingError, RequestError
+from glide.exceptions import ClosingError
 from glide.glide_client import GlideClient, GlideClusterClient, TGlideClient
 from glide.logger import Level as logLevel
 from glide.logger import Logger

python/requirements.txt

@@ -1,8 +1,7 @@
 async-timeout==4.0.2;python_version<"3.11"
-maturin==0.13.0
+maturin==0.14.17 # higher version break the needs structure changes, the name of the project is not the same as the package name, and the naming both glide create a circular dependency - TODO: fix this 
 protobuf==3.20.*
 pytest
 pytest-asyncio
 typing_extensions==4.8.0;python_version<"3.11"
 pytest-html
-pyrsistent