From a3f7d846ed73596a4b875e10323a05e46fbe4214 Mon Sep 17 00:00:00 2001 From: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> Date: Sat, 19 Oct 2024 15:13:05 +0300 Subject: [PATCH 1/8] Update README.md nodejs platform support (#2397) * Update README.md Signed-off-by: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> * Update README.md - lint fix Signed-off-by: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> * Update README.md ling Signed-off-by: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> * Update node/README.md Co-authored-by: Yury-Fridlyand Signed-off-by: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> --------- Signed-off-by: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> Co-authored-by: Yury-Fridlyand --- node/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/node/README.md b/node/README.md index d215bc102f..661e742b96 100644 --- a/node/README.md +++ b/node/README.md @@ -14,12 +14,16 @@ The release of Valkey GLIDE was tested on the following platforms: Linux: -- Ubuntu 22.04.1 (x86_64) +- Ubuntu 22.04.1 (x86_64 and aarch64) - Amazon Linux 2023 (AL2023) (x86_64) macOS: -- macOS 12.7 (Apple silicon/aarch_64 and Intel/x86_64) +- macOS (12.7 and latest) (Apple silicon/aarch_64 and Intel/x86_64) + +Alpine: + +- node:alpine (default on aarch64 and x86_64) ## NodeJS supported version From fee8d293ee76f7eb34dce28ce73f29ce9828ed29 Mon Sep 17 00:00:00 2001 From: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:44:16 +0300 Subject: [PATCH 2/8] CI - Minimal and full CI matrix impl (#2051) * CI - Minimal and full CI matrix impl Signed-off-by: avifenesh * Fix mypy failing (#2453) --------- Signed-off-by: Shoham Elias Signed-off-by: avifenesh * Python: adds JSON.ARRLEN command (#2403) --------- Signed-off-by: Shoham Elias Signed-off-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> Signed-off-by: avifenesh --------- Signed-off-by: avifenesh Signed-off-by: Shoham Elias Signed-off-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> Co-authored-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> --- .github/DEVELOPER.md | 132 ++++++++++ .github/ISSUE_TEMPLATE/bug-report.yml | 236 +++++++++--------- .github/ISSUE_TEMPLATE/feature-request.yml | 100 ++++---- .github/json_matrices/build-matrix.json | 31 +++ .github/json_matrices/engine-matrix.json | 26 +- .../supported-languages-versions.json | 17 ++ .github/pull_request_template.md | 14 +- .../workflows/build-node-wrapper/action.yml | 14 +- .github/workflows/codeql.yml | 12 + .../workflows/create-test-matrices/action.yml | 54 ++++ .github/workflows/csharp.yml | 23 +- .github/workflows/go.yml | 52 ++-- .github/workflows/install-redis/action.yml | 2 +- .../install-rust-and-protoc/action.yml | 1 - .../install-shared-dependencies/action.yml | 13 +- .github/workflows/install-valkey/action.yml | 72 +++--- .github/workflows/java-cd.yml | 19 +- .github/workflows/java.yml | 64 +++-- .github/workflows/lint-rust/action.yml | 2 +- .github/workflows/lint-ts/action.yml | 2 +- .../node-create-package-file/action.yml | 2 +- .github/workflows/node.yml | 107 ++++---- .github/workflows/npm-cd.yml | 111 ++++---- .github/workflows/ort.yml | 187 +++++++------- .github/workflows/pypi-cd.yml | 11 + .github/workflows/python.yml | 159 +++++++----- .github/workflows/run-ort-tools/action.yml | 16 +- .github/workflows/rust.yml | 19 +- .github/workflows/semgrep.yml | 14 ++ .../workflows/setup-musl-on-linux/action.yml | 15 +- .../start-self-hosted-runner/action.yml | 6 +- .prettierignore | 1 + .vscode/settings.json | 2 +- CHANGELOG.md | 4 + node/tests/GlideClient.test.ts | 48 +++- node/tests/GlideClusterClient.test.ts | 32 ++- .../async_commands/server_modules/json.py | 54 ++++ 37 files changed, 1069 insertions(+), 605 deletions(-) create mode 100644 .github/DEVELOPER.md create mode 100644 .github/json_matrices/supported-languages-versions.json create mode 100644 .github/workflows/create-test-matrices/action.yml create mode 100644 .prettierignore diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md new file mode 100644 index 0000000000..710bd08dca --- /dev/null +++ b/.github/DEVELOPER.md @@ -0,0 +1,132 @@ +# CI/CD Workflow Guide + +TODO: Add a description of the CI/CD workflow and its components. + +### Overview + +Our CI/CD pipeline tests and builds our project across multiple languages, versions, and environments. This guide outlines the key components and processes of our workflow. + +### Workflow Triggers + +- Push to `main` branch +- Pull requests +- Scheduled runs (daily) +- Manual trigger (workflow_dispatch) + +### Language-Specific Workflows + +Each language has its own workflow file with similar structure but language-specific steps, for example python.yml for Python, or java.yml for Java. + +### Shared Components + +#### Matrix Files + +While workflows are language-specific, the matrix files are shared across all workflows. +Workflows are starting by loading the matrix files from the `.github/json_matrices` directory. + +- `engine-matrix.json`: Defines the versions of Valkey engine to test against. +- `build-matrix.json`: Defines the host environments for testing. +- `supported-languages-version.json`: Defines the supported versions of languages. + +All matrices have a `run` like field which specifies if the configuration should be tested on every workflow run. +This allows for flexible control over which configurations are tested in different scenarios, optimizing CI/CD performance and resource usage. + +#### Engine Matrix (engine-matrix.json) + +Defines the versions of Valkey engine to test against: + +```json +[{ "type": "valkey", "version": "7.2.5", "run": "always" }] +``` + +- `type`: The type of engine (e.g., Valkey, Redis). +- `version`: The version of the engine. +- `run`: Specifies if the engine version should be tested on every workflow. + +#### Build Matrix (build-matrix.json) + +Defines the host environments for testing: + +```json +[ + { + "OS": "ubuntu", + "RUNNER": "ubuntu-latest", + "TARGET": "x86_64-unknown-linux-gnu", + "run": ["always", "python", "node", "java"] + } + // ... other configurations +] +``` + +- `OS`: The operating system of the host. +- `RUNNER`: The GitHub runner to use. +- `TARGET`: The target environment. +- `run`: Specifies which language workflows should use this host configuration, always means run on each workflow trigger. + +#### Supported Languages Version (supported-languages-version.json) + +Defines the supported versions of languages: + +```json +[ + { + "language": "java", + "versions": ["11", "17"], + "always-run-versions": ["17"] + } + // ... other configurations +] +``` + +- `language`: The language for which the version is supported. +- `versions`: The full versions supported of the language which will test against scheduled. +- `always-run-versions`: The versions which will be tested in every workflow run. + +#### Triggering Workflows + +Push to main or create a pull request to run workflows automatically. +Use workflow_dispatch for manual triggers, accepting inputs of full-matrix which is a boolean value to run all configurations. +Scheduled runs are triggered daily to ensure regular testing of all configurations. + +### Mutual vs. Language-Specific Components + +#### Mutual + +`Matrix files` - `.github/json_matrices` +`Shared dependencies installation` - `.github/workflows/install-shared-dependencies/action.yml` +`Linting Rust` - `.github/workflows/lint-rust/action.yml` + +#### Language-Specific + +`Package manager commands` +`Testing frameworks` +`Build processes` + +### Customizing Workflows + +Modify `[language].yml` files to adjust language-specific steps. +Update matrix files to change tested versions or environments. +Adjust cron schedules in workflow files for different timing of scheduled runs. + +### Workflow Matrices + +We use dynamic matrices for our CI/CD workflows, which are created using the `create-test-matrices` action. This action is defined in `.github/workflows/create-test-matrices/action.yml`. + +#### How it works + +1. The action is called with a `language-name` input and `dispatch-run-full-matrix` input. +2. It reads the `engine-matrix.json`, `build-matrix.json`, and `supported-languages-version.json` files. +3. It filters the matrices based on the inputs and the event type. +4. It generates three matrices: + - Engine matrix: Defines the types and versions of the engine to test against, for example Valkey 7.2.5. + - Host matrix: Defines the host platforms to run the tests on, for example Ubuntu on ARM64. + - Language-version matrix: Defines the supported versions of languages, for example python 3.8. + +#### Outputs + +- `engine-matrix-output`: The generated engine matrix. +- `host-matrix-output`: The generated host matrix. +- `language-version-matrix-output`: The generated language version matrix. + +This dynamic matrix generation allows for flexible and efficient CI/CD workflows, adapting the test configurations based on the type of change and the specific language being tested. diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 5f63253d51..98a018219c 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -5,134 +5,134 @@ title: "(topic): (short issue description)" labels: [bug, needs-triage] assignees: [] body: - - type: textarea - id: description - attributes: - label: Describe the bug - description: What is the problem? A clear and concise description of the bug. - validations: - required: true + - type: textarea + id: description + attributes: + label: Describe the bug + description: What is the problem? A clear and concise description of the bug. + validations: + required: true - - type: textarea - id: expected - attributes: - label: Expected Behavior - description: | - What did you expect to happen? - validations: - required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: | + What did you expect to happen? + validations: + required: true - - type: textarea - id: current - attributes: - label: Current Behavior - description: | - What actually happened? - - Please include full errors, uncaught exceptions, stack traces, and relevant logs. - If service responses are relevant, please include wire logs. - validations: - required: true + - type: textarea + id: current + attributes: + label: Current Behavior + description: | + What actually happened? - - type: textarea - id: reproduction - attributes: - label: Reproduction Steps - description: | - Provide a self-contained, concise snippet of code that can be used to reproduce the issue. - For more complex issues provide a repo with the smallest sample that reproduces the bug. - - Avoid including business logic or unrelated code, it makes diagnosis more difficult. - The code sample should be an SSCCE. See http://sscce.org/ for details. In short, please provide a code sample that we can copy/paste, run and reproduce. - validations: - required: true + Please include full errors, uncaught exceptions, stack traces, and relevant logs. + If service responses are relevant, please include wire logs. + validations: + required: true - - type: textarea - id: solution - attributes: - label: Possible Solution - description: | - Suggest a fix/reason for the bug - validations: - required: false + - type: textarea + id: reproduction + attributes: + label: Reproduction Steps + description: | + Provide a self-contained, concise snippet of code that can be used to reproduce the issue. + For more complex issues provide a repo with the smallest sample that reproduces the bug. - - type: textarea - id: context - attributes: - label: Additional Information/Context - description: | - Anything else that might be relevant for troubleshooting this bug. Providing context helps us come up with a solution that is most useful in the real world. - validations: - required: false + Avoid including business logic or unrelated code, it makes diagnosis more difficult. + The code sample should be an SSCCE. See http://sscce.org/ for details. In short, please provide a code sample that we can copy/paste, run and reproduce. + validations: + required: true - - type: input - id: client-version - attributes: - label: Client version used - validations: - required: true + - type: textarea + id: solution + attributes: + label: Possible Solution + description: | + Suggest a fix/reason for the bug + validations: + required: false - - type: input - id: engine-version - attributes: - label: Engine type and version - description: E.g. Valkey 7.0 - validations: - required: true + - type: textarea + id: context + attributes: + label: Additional Information/Context + description: | + Anything else that might be relevant for troubleshooting this bug. Providing context helps us come up with a solution that is most useful in the real world. + validations: + required: false - - type: input - id: operating-system - attributes: - label: OS - validations: - required: true + - type: input + id: client-version + attributes: + label: Client version used + validations: + required: true - - type: dropdown - id: language - attributes: - label: Language - multiple: true - options: - - TypeScript - - Python - - Java - - Rust - - Go - - .Net - validations: - required: true + - type: input + id: engine-version + attributes: + label: Engine type and version + description: E.g. Valkey 7.0 + validations: + required: true - - type: input - id: language-version - attributes: - label: Language Version - description: E.g. TypeScript (5.2.2) | Python (3.9) - validations: - required: true + - type: input + id: operating-system + attributes: + label: OS + validations: + required: true - - type: textarea - id: cluster-info - attributes: - label: Cluster information - description: | - Cluster information, cluster topology, number of shards, number of replicas, used data types. - validations: - required: false + - type: dropdown + id: language + attributes: + label: Language + multiple: true + options: + - TypeScript + - Python + - Java + - Rust + - Go + - .Net + validations: + required: true - - type: textarea - id: logs - attributes: - label: Logs - description: | - Client and/or server logs. - validations: - required: false + - type: input + id: language-version + attributes: + label: Language Version + description: E.g. TypeScript (5.2.2) | Python (3.9) + validations: + required: true - - type: textarea - id: other - attributes: - label: Other information - description: | - e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. associated pull-request, stackoverflow, etc - validations: - required: false + - type: textarea + id: cluster-info + attributes: + label: Cluster information + description: | + Cluster information, cluster topology, number of shards, number of replicas, used data types. + validations: + required: false + + - type: textarea + id: logs + attributes: + label: Logs + description: | + Client and/or server logs. + validations: + required: false + + - type: textarea + id: other + attributes: + label: Other information + description: | + e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, eg. associated pull-request, stackoverflow, etc + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 64684f1d1c..a7607565aa 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -5,55 +5,55 @@ title: "(topic): (short issue description)" labels: [feature-request, needs-triage] assignees: [] body: - - type: textarea - id: description - attributes: - label: Describe the feature - description: A clear and concise description of the feature you are proposing. - validations: - required: true - - type: textarea - id: use-case - attributes: - label: Use Case - description: | - Why do you need this feature? - validations: - required: true - - type: textarea - id: solution - attributes: - label: Proposed Solution - description: | - Suggest how to implement the addition or change. Please include prototype/workaround/sketch/reference implementation. - validations: - required: false - - type: textarea - id: other - attributes: - label: Other Information - description: | - Any alternative solutions or features you considered, a more detailed explanation, stack traces, related issues, links for context, etc. - validations: - required: false - - type: checkboxes - id: ack - attributes: - label: Acknowledgements - options: - - label: I may be able to implement this feature request + - type: textarea + id: description + attributes: + label: Describe the feature + description: A clear and concise description of the feature you are proposing. + validations: + required: true + - type: textarea + id: use-case + attributes: + label: Use Case + description: | + Why do you need this feature? + validations: + required: true + - type: textarea + id: solution + attributes: + label: Proposed Solution + description: | + Suggest how to implement the addition or change. Please include prototype/workaround/sketch/reference implementation. + validations: required: false - - label: This feature might incur a breaking change + - type: textarea + id: other + attributes: + label: Other Information + description: | + Any alternative solutions or features you considered, a more detailed explanation, stack traces, related issues, links for context, etc. + validations: required: false - - type: input - id: client-version - attributes: - label: Client version used - validations: - required: true - - type: input - id: environment - attributes: - label: Environment details (OS name and version, etc.) - validations: - required: true + - type: checkboxes + id: ack + attributes: + label: Acknowledgements + options: + - label: I may be able to implement this feature request + required: false + - label: This feature might incur a breaking change + required: false + - type: input + id: client-version + attributes: + label: Client version used + validations: + required: true + - type: input + id: environment + attributes: + label: Environment details (OS name and version, etc.) + validations: + required: true diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index fc02093b9f..e91aeeefe3 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -5,11 +5,16 @@ "RUNNER": "ubuntu-latest", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", +<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ] +======= + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["always", "python", "node", "java"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "ubuntu", @@ -21,12 +26,18 @@ ], "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-gnu", +<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ], "CONTAINER": "2_28" +======= + "PACKAGE_MANAGERS": ["pypi", "npm"], + "CONTAINER": "2_28", + "run": ["python", "node", "java"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "macos", @@ -34,11 +45,16 @@ "RUNNER": "macos-12", "ARCH": "x64", "TARGET": "x86_64-apple-darwin", +<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ] +======= + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["python", "node", "java"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "macos", @@ -46,11 +62,16 @@ "RUNNER": "macos-latest", "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", +<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ] +======= + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["python", "node", "java"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "ubuntu", @@ -64,9 +85,14 @@ ], "IMAGE": "node:lts-alpine3.19", "CONTAINER_OPTIONS": "--user root --privileged --rm", +<<<<<<< HEAD "PACKAGE_MANAGERS": [ "npm" ] +======= + "PACKAGE_MANAGERS": ["npm"], + "run": ["node"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "ubuntu", @@ -76,8 +102,13 @@ "RUNNER": "ubuntu-latest", "IMAGE": "node:lts-alpine3.19", "CONTAINER_OPTIONS": "--user root --privileged", +<<<<<<< HEAD "PACKAGE_MANAGERS": [ "npm" ] +======= + "PACKAGE_MANAGERS": ["npm"], + "run": ["node"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) } ] diff --git a/.github/json_matrices/engine-matrix.json b/.github/json_matrices/engine-matrix.json index 464aedf31a..149f6f1080 100644 --- a/.github/json_matrices/engine-matrix.json +++ b/.github/json_matrices/engine-matrix.json @@ -1,10 +1,20 @@ [ - { - "type": "valkey", - "version": "7.2.5" - }, - { - "type": "valkey", - "version": "8.0.0" - } + { + "type": "valkey", + "version": "8.0.0", + "run": "always" + }, + { + "type": "valkey", + "version": "7.2.5" + }, + { + "type": "redis", + "version": "7.0" + }, + { + "type": "redis", + "version": "6.2", + "run": "always" + } ] diff --git a/.github/json_matrices/supported-languages-versions.json b/.github/json_matrices/supported-languages-versions.json new file mode 100644 index 0000000000..bc6fd5f472 --- /dev/null +++ b/.github/json_matrices/supported-languages-versions.json @@ -0,0 +1,17 @@ +[ + { + "language": "java", + "versions": ["11", "17"], + "always-run-versions": ["17"] + }, + { + "language": "python", + "versions": ["3.8", "3.9", "3.10", "3.11", "3.12"], + "always-run-versions": ["3.8", "3.12"] + }, + { + "language": "node", + "full-versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], + "always-run-versions": ["16.x", "20.x"] + } +] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8e6d8dd2b3..ff120235e3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,16 +7,16 @@ here](https://github.com/valkey-io/valkey-glide/blob/main/CONTRIBUTING.md) --> ### Issue link + This Pull Request is linked to issue (URL): [REPLACE ME] ### Checklist Before submitting the PR make sure the following are checked: -* [ ] This Pull Request is related to one issue. -* [ ] Commit message has a detailed description of what changed and why. -* [ ] Tests are added or updated. -* [ ] CHANGELOG.md and documentation files are updated. -* [ ] Destination branch is correct - main or release -* [ ] Commits will be squashed upon merging. - +- [ ] This Pull Request is related to one issue. +- [ ] Commit message has a detailed description of what changed and why. +- [ ] Tests are added or updated. +- [ ] CHANGELOG.md and documentation files are updated. +- [ ] Destination branch is correct - main or release +- [ ] Commits will be squashed upon merging. diff --git a/.github/workflows/build-node-wrapper/action.yml b/.github/workflows/build-node-wrapper/action.yml index aa1200fbd5..f7590c8b0a 100644 --- a/.github/workflows/build-node-wrapper/action.yml +++ b/.github/workflows/build-node-wrapper/action.yml @@ -65,13 +65,13 @@ runs: - name: Create package.json file uses: ./.github/workflows/node-create-package-file with: - release_version: ${{ env.RELEASE_VERSION }} - os: ${{ inputs.os }} - named_os: ${{ inputs.named_os }} - arch: ${{ inputs.arch }} - npm_scope: ${{ inputs.npm_scope }} - target: ${{ inputs.target }} - + release_version: ${{ env.RELEASE_VERSION }} + os: ${{ inputs.os }} + named_os: ${{ inputs.named_os }} + arch: ${{ inputs.arch }} + npm_scope: ${{ inputs.npm_scope }} + target: ${{ inputs.target }} + - name: npm install shell: bash working-directory: ./node diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 36ac59f664..37ccbf1c2f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,21 +2,33 @@ name: "CodeQL" on: push: +<<<<<<< HEAD branches: +======= + branches: +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - "main" - "v.?[0-9]+.[0-9]+.[0-9]+" - "v.?[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+" +<<<<<<< HEAD - release-* pull_request: branches: +======= + pull_request: + branches: +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - "main" - "v.?[0-9]+.[0-9]+.[0-9]+" - "v.?[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+" +<<<<<<< HEAD - release-* +======= +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) schedule: - cron: "37 18 * * 6" diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml new file mode 100644 index 0000000000..61516b1cc9 --- /dev/null +++ b/.github/workflows/create-test-matrices/action.yml @@ -0,0 +1,54 @@ +inputs: + language-name: + description: "Language name" + required: true + dispatch-run-full-matrix: + description: "Run the full matrix" + required: false + default: "false" +outputs: + engine-matrix-output: + description: "Engine matrix" + value: ${{ steps.load-engine-matrix.outputs.engine-matrix }} + host-matrix-output: + description: "Host matrix" + value: ${{ steps.load-host-matrix.outputs.host-matrix }} + version-matrix-output: + description: "Version matrix" + value: ${{ steps.create-version-matrix.outputs.version-matrix }} + +runs: + using: "composite" + steps: + - name: Load engine matrix + id: load-engine-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then + echo "engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + echo "engine-matrix=$(cat $GITHUB_OUTPUT)" + + - name: Load host matrix + id: load-host-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then + echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["always"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + else + echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["${{ inputs.language-name }}"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + fi + echo "host-matrix=$(cat $GITHUB_OUTPUT)" + + - name: Create version matrix + id: create-version-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then + echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + else + echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | (.versions // .["full-versions"]) | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + fi + echo "version-matrix=$(cat $GITHUB_OUTPUT)" diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index eab61c6dc1..c7de56e9ed 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -40,16 +40,16 @@ jobs: load-engine-matrix: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - + - name: Load the engine matrix id: load-engine-matrix shell: bash run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - + run-tests: needs: load-engine-matrix timeout-minutes: 25 @@ -59,19 +59,19 @@ jobs: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} dotnet: # - '6.0' - - '8.0' + - "8.0" host: - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } + OS: ubuntu, + RUNNER: ubuntu-latest, + TARGET: x86_64-unknown-linux-gnu, + } # - { # OS: macos, # RUNNER: macos-latest, # TARGET: aarch64-apple-darwin # } - + runs-on: ${{ matrix.host.RUNNER }} steps: @@ -82,7 +82,7 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ matrix.dotnet }} - + - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: @@ -114,7 +114,7 @@ jobs: benchmarks/results/* utils/clusters/** -# TODO Add amazonlinux + # TODO Add amazonlinux lint-rust: timeout-minutes: 10 @@ -126,3 +126,4 @@ jobs: - uses: ./.github/workflows/lint-rust with: cargo-toml-folder: ./csharp/lib + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3290839a6a..95373ffc09 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,10 +2,14 @@ name: Go CI on: push: +<<<<<<< HEAD branches: - main - release-* - v* +======= + branches: ["main"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) paths: - glide-core/src/** - glide-core/redis-rs/redis/src/** @@ -35,17 +39,17 @@ concurrency: jobs: load-engine-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT build-and-test-go-client: needs: load-engine-matrix @@ -55,19 +59,19 @@ jobs: fail-fast: false matrix: go: - - '1.22.0' + - "1.22.0" engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } + - { + OS: ubuntu, + RUNNER: ubuntu-latest, + TARGET: x86_64-unknown-linux-gnu, + } + # - { + # OS: macos, + # RUNNER: macos-latest, + # TARGET: aarch64-apple-darwin + # } runs-on: ${{ matrix.host.RUNNER }} @@ -108,7 +112,7 @@ jobs: - name: Run tests working-directory: ./go run: | - make test + make test - uses: ./.github/workflows/test-benchmark with: @@ -187,7 +191,7 @@ jobs: - name: Run tests working-directory: ./go run: | - make test + make test - name: Upload cluster manager logs if: always() diff --git a/.github/workflows/install-redis/action.yml b/.github/workflows/install-redis/action.yml index b60f0687b5..8c8d707aa0 100644 --- a/.github/workflows/install-redis/action.yml +++ b/.github/workflows/install-redis/action.yml @@ -35,7 +35,7 @@ runs: wget https://github.com/redis/redis/archive/${{ inputs.redis-version }}.tar.gz; tar -xzvf ${{ inputs.redis-version }}.tar.gz; pushd redis-${{ inputs.redis-version }} && BUILD_TLS=yes make && sudo mv src/redis-server src/redis-cli ~/redis-binaries/${{ inputs.redis-version }} && popd; - + - name: Remove the source package shell: bash if: steps.cache-redis.outputs.cache-hit != 'true' diff --git a/.github/workflows/install-rust-and-protoc/action.yml b/.github/workflows/install-rust-and-protoc/action.yml index e1222ffd9d..31987ba04a 100644 --- a/.github/workflows/install-rust-and-protoc/action.yml +++ b/.github/workflows/install-rust-and-protoc/action.yml @@ -16,7 +16,6 @@ inputs: type: string required: true - runs: using: "composite" steps: diff --git a/.github/workflows/install-shared-dependencies/action.yml b/.github/workflows/install-shared-dependencies/action.yml index ed065e9840..ac75e48596 100644 --- a/.github/workflows/install-shared-dependencies/action.yml +++ b/.github/workflows/install-shared-dependencies/action.yml @@ -23,14 +23,19 @@ inputs: - x86_64-unknown-linux-musl engine-version: description: "Engine version to install" +<<<<<<< HEAD required: false type: string +======= + required: true + type: string + +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) github-token: description: "GITHUB_TOKEN, GitHub App installation access token" required: true type: string - runs: using: "composite" steps: @@ -47,7 +52,7 @@ runs: run: | sudo apt update -y sudo apt install -y git gcc pkg-config openssl libssl-dev - + - name: Install software dependencies for Ubuntu MUSL shell: bash if: "${{ contains(inputs.target, 'musl') }}" @@ -67,8 +72,8 @@ runs: if: "${{ !contains(inputs.target, 'musl') }}" uses: ./.github/workflows/install-rust-and-protoc with: - target: ${{ inputs.target }} - github-token: ${{ inputs.github-token }} + target: ${{ inputs.target }} + github-token: ${{ inputs.github-token }} - name: Install Valkey if: ${{ inputs.engine-version != '' }} diff --git a/.github/workflows/install-valkey/action.yml b/.github/workflows/install-valkey/action.yml index 74c75572a4..36da82f93b 100644 --- a/.github/workflows/install-valkey/action.yml +++ b/.github/workflows/install-valkey/action.yml @@ -25,55 +25,55 @@ runs: using: "composite" steps: - - name: Cache Valkey - # TODO: remove the musl ARM64 limitation when https://github.com/actions/runner/issues/801 is resolved + - name: + Cache Valkey + # TODO: remove the musl ARM64 limitation when https://github.com/actions/runner/issues/801 is resolved if: ${{ inputs.target != 'aarch64-unknown-linux-musl' }} uses: actions/cache@v4 id: cache-valkey with: - path: | - ~/valkey - key: valkey-${{ inputs.engine-version }}-${{ inputs.target }} + path: | + ~/valkey + key: valkey-${{ inputs.engine-version }}-${{ inputs.target }} - name: Build Valkey if: ${{ steps.cache-valkey.outputs.cache-hit != 'true' }} shell: bash - run: | - echo "Building valkey ${{ inputs.engine-version }}" - cd ~ - rm -rf valkey - git clone https://github.com/valkey-io/valkey.git - cd valkey - git checkout ${{ inputs.engine-version }} - make BUILD_TLS=yes + run: | + echo "Building valkey ${{ inputs.engine-version }}" + cd ~ + rm -rf valkey + git clone https://github.com/valkey-io/valkey.git + cd valkey + git checkout ${{ inputs.engine-version }} + make BUILD_TLS=yes - name: Install Valkey shell: bash run: | - cd ~/valkey - if command -v sudo &> /dev/null - then - echo "sudo command exists" - sudo make install - else - echo "sudo command does not exist" - make install - fi - echo 'export PATH=/usr/local/bin:$PATH' >>~/.bash_profile + cd ~/valkey + if command -v sudo &> /dev/null + then + echo "sudo command exists" + sudo make install + else + echo "sudo command does not exist" + make install + fi + echo 'export PATH=/usr/local/bin:$PATH' >>~/.bash_profile - name: Verify Valkey installation and symlinks if: ${{ !contains(inputs.engine-version, '-rc') }} shell: bash - run: | - # In Valkey releases, the engine is built with symlinks from valkey-server and valkey-cli - # to redis-server and redis-cli. This step ensures that the engine is properly installed - # with the expected version and that Valkey symlinks are correctly created. - EXPECTED_VERSION=`echo ${{ inputs.engine-version }} | sed -e "s/^redis-//"` - INSTALLED_VER=$(redis-server -v) - if [[ $INSTALLED_VER != *"${EXPECTED_VERSION}"* ]]; then - echo "Wrong version has been installed. Expected: $EXPECTED_VERSION, Installed: $INSTALLED_VER" - exit 1 - else - echo "Successfully installed the server: $INSTALLED_VER" - fi - + run: | + # In Valkey releases, the engine is built with symlinks from valkey-server and valkey-cli + # to redis-server and redis-cli. This step ensures that the engine is properly installed + # with the expected version and that Valkey symlinks are correctly created. + EXPECTED_VERSION=`echo ${{ inputs.engine-version }} | sed -e "s/^redis-//"` + INSTALLED_VER=$(redis-server -v) + if [[ $INSTALLED_VER != *"${EXPECTED_VERSION}"* ]]; then + echo "Wrong version has been installed. Expected: $EXPECTED_VERSION, Installed: $INSTALLED_VER" + exit 1 + else + echo "Successfully installed the server: $INSTALLED_VER" + fi diff --git a/.github/workflows/java-cd.yml b/.github/workflows/java-cd.yml index d3f2038313..3263dd3a6f 100644 --- a/.github/workflows/java-cd.yml +++ b/.github/workflows/java-cd.yml @@ -211,7 +211,12 @@ jobs: exit 1 test-deployment-on-all-architectures: - needs: [set-release-version, load-platform-matrix, publish-to-maven-central-deployment] + needs: + [ + set-release-version, + load-platform-matrix, + publish-to-maven-central-deployment, + ] env: JAVA_VERSION: "11" RELEASE_VERSION: ${{ needs.set-release-version.outputs.RELEASE_VERSION }} @@ -265,7 +270,11 @@ jobs: publish-release-to-maven: if: ${{ inputs.maven_publish == true || github.event_name == 'push' }} - needs: [publish-to-maven-central-deployment, test-deployment-on-all-architectures] + needs: + [ + publish-to-maven-central-deployment, + test-deployment-on-all-architectures, + ] runs-on: ubuntu-latest environment: AWS_ACTIONS env: @@ -279,7 +288,11 @@ jobs: drop-deployment-if-validation-fails: if: ${{ failure() }} - needs: [publish-to-maven-central-deployment, test-deployment-on-all-architectures] + needs: + [ + publish-to-maven-central-deployment, + test-deployment-on-all-architectures, + ] runs-on: ubuntu-latest env: DEPLOYMENT_ID: ${{ needs.publish-to-maven-central-deployment.outputs.DEPLOYMENT_ID }} diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 2c31562f78..af792539f0 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -28,48 +28,46 @@ on: - .github/workflows/install-valkey/action.yml - .github/json_matrices/build-matrix.json workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: false + default: "false" + + schedule: + - cron: "0 0 * * *" concurrency: group: java-${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: java + dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} - build-and-test-java-client: - needs: load-engine-matrix + test-java: + needs: get-matrices timeout-minutes: 35 strategy: # Run all jobs fail-fast: false matrix: - java: - # - 11 - - 17 - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu, - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -124,14 +122,14 @@ jobs: java/client/build/reports/spotbugs/** build-amazonlinux-latest: - if: github.repository_owner == 'valkey-io' + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') + needs: get-matrices strategy: # Run all jobs fail-fast: false matrix: - java: - # - 11 - - 17 + java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} runs-on: ubuntu-latest container: amazonlinux:latest timeout-minutes: 35 @@ -158,7 +156,7 @@ jobs: os: "amazon-linux" target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" + engine-version: ${{ matrix.engine.version }} - name: Install protoc (protobuf) uses: arduino/setup-protoc@v3 @@ -179,7 +177,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-reports-${{ matrix.java }}-amazon-linux + name: test-reports-${{ matrix.java }}-${{ matrix.engine }}-amazon-linux path: | java/client/build/reports/** java/integTest/build/reports/** diff --git a/.github/workflows/lint-rust/action.yml b/.github/workflows/lint-rust/action.yml index 06b0b7a75a..198c04de1c 100644 --- a/.github/workflows/lint-rust/action.yml +++ b/.github/workflows/lint-rust/action.yml @@ -35,7 +35,7 @@ runs: # We run clippy without features - run: cargo clippy --all-targets -- -D warnings working-directory: ${{ inputs.cargo-toml-folder }} - shell: bash + shell: bash - run: | cargo update diff --git a/.github/workflows/lint-ts/action.yml b/.github/workflows/lint-ts/action.yml index 834e3d7ec9..2f4b47e358 100644 --- a/.github/workflows/lint-ts/action.yml +++ b/.github/workflows/lint-ts/action.yml @@ -12,7 +12,7 @@ runs: steps: - uses: actions/checkout@v4 - - run: cp eslint.config.mjs ${{ inputs.package-folder }} + - run: cp eslint.config.mjs ${{ inputs.package-folder }} shell: bash - run: | diff --git a/.github/workflows/node-create-package-file/action.yml b/.github/workflows/node-create-package-file/action.yml index 1da7510aab..8c9cc9d2f2 100644 --- a/.github/workflows/node-create-package-file/action.yml +++ b/.github/workflows/node-create-package-file/action.yml @@ -84,4 +84,4 @@ runs: mv package.json package.json.tmpl envsubst < package.json.tmpl > "package.json" cat package.json - echo $(ls *json*) + echo $(ls *json*) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 1927649247..e02ef98806 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -34,6 +34,14 @@ on: - .github/json_matrices/build-matrix.json - .github/workflows/start-self-hosted-runner/action.yml workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: false + default: "false" + + schedule: + - cron: "0 1 * * *" concurrency: group: node-${{ github.head_ref || github.ref }} @@ -43,6 +51,7 @@ env: CARGO_TERM_COLOR: always jobs: +<<<<<<< HEAD load-engine-matrix: runs-on: ubuntu-latest outputs: @@ -57,28 +66,56 @@ jobs: run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT test-ubuntu-latest: +======= + get-matrices: +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) + runs-on: ubuntu-latest + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: node + dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + + test-node: runs-on: ubuntu-latest - needs: load-engine-matrix + needs: [get-matrices] timeout-minutes: 25 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} + node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output)}} steps: - uses: actions/checkout@v4 +<<<<<<< HEAD - name: Use Node.js 16.x uses: actions/setup-node@v4 +======= + - name: Setup Node + uses: actions/setup-node@v4 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) with: - node-version: 16.x + node-version: ${{ matrix.node }} - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: - os: "ubuntu" - target: "x86_64-unknown-linux-gnu" + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} @@ -113,7 +150,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-ubuntu + name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.node }}-ubuntu path: | node/test-report*.html utils/clusters/** @@ -132,52 +169,8 @@ jobs: cargo-toml-folder: ./node/rust-client github-token: ${{ secrets.GITHUB_TOKEN }} - # build-macos-latest: - # runs-on: macos-latest - # timeout-minutes: 25 - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: recursive - # - name: Set up Homebrew - # uses: Homebrew/actions/setup-homebrew@master - - # - name: Install NodeJS - # run: | - # brew update - # brew upgrade || true - # brew install node - - # - name: Downgrade npm major version to 8 - # run: | - # npm i -g npm@8 - - # - name: Build Node wrapper - # uses: ./.github/workflows/build-node-wrapper - # with: - # os: "macos" - # named_os: "darwin" - # arch: "arm64" - # target: "aarch64-apple-darwin" - # github-token: ${{ secrets.GITHUB_TOKEN }} - # engine-version: "7.2.5" - - # - name: Test compatibility - # run: npm test -- -t "set and get flow works" - # working-directory: ./node - - # - name: Upload test reports - # if: always() - # continue-on-error: true - # uses: actions/upload-artifact@v4 - # with: - # name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-macos - # path: | - # node/test-report*.html - # utils/clusters/** - # benchmarks/results/** - build-amazonlinux-latest: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') runs-on: ubuntu-latest container: amazonlinux:latest timeout-minutes: 15 @@ -209,8 +202,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - - name: Test compatibility - run: npm test -- -t "set and get flow works" + - name: test + run: npm test working-directory: ./node - name: Upload test reports @@ -225,6 +218,7 @@ jobs: benchmarks/results/** build-and-test-linux-musl-on-x86: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') name: Build and test Node wrapper on Linux musl runs-on: ubuntu-latest container: @@ -257,9 +251,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - - name: Test compatibility - shell: bash - run: npm test -- -t "set and get flow works" + - name: test + run: npm test working-directory: ./node - name: Upload test reports diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 362117affb..83011739b4 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -4,29 +4,29 @@ name: NPM - Continuous Deployment on: pull_request: - paths: - - .github/workflows/npm-cd.yml - - .github/workflows/build-node-wrapper/action.yml - - .github/workflows/start-self-hosted-runner/action.yml - - .github/workflows/install-rust-and-protoc/action.yml - - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + paths: + - .github/workflows/npm-cd.yml + - .github/workflows/build-node-wrapper/action.yml + - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/install-rust-and-protoc/action.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json push: tags: - "v*.*" workflow_dispatch: - inputs: - version: - description: 'The release version of GLIDE, formatted as *.*.* or *.*.*-rc*' - required: true + inputs: + version: + description: "The release version of GLIDE, formatted as *.*.* or *.*.*-rc*" + required: true concurrency: group: npm-${{ github.head_ref || github.ref }} cancel-in-progress: true permissions: - id-token: write + id-token: write jobs: start-self-hosted-runner: @@ -34,17 +34,17 @@ jobs: runs-on: ubuntu-latest environment: AWS_ACTIONS steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Start self hosted EC2 runner - uses: ./.github/workflows/start-self-hosted-runner - with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Start self hosted EC2 runner + uses: ./.github/workflows/start-self-hosted-runner + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} load-platform-matrix: runs-on: ubuntu-latest @@ -58,9 +58,9 @@ jobs: id: load-platform-matrix shell: bash run: | - # Get the matrix from the matrix.json file, without the object that has the IMAGE key - export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["npm"])))' < .github/json_matrices/build-matrix.json | jq -c .)" - echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT + # Get the matrix from the matrix.json file, without the object that has the IMAGE key + export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["npm"])))' < .github/json_matrices/build-matrix.json | jq -c .)" + echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT publish-binaries: needs: [start-self-hosted-runner, load-platform-matrix] @@ -73,7 +73,7 @@ jobs: strategy: fail-fast: false matrix: - build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} + build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} steps: - name: Setup self-hosted runner access if: ${{ contains(matrix.build.RUNNER, 'self-hosted') && matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} @@ -85,7 +85,7 @@ jobs: run: | apk update apk add git - + - name: Checkout if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} uses: actions/checkout@v4 @@ -100,7 +100,7 @@ jobs: workspace: $GITHUB_WORKSPACE npm-scope: ${{ vars.NPM_SCOPE }} npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} - arch: ${{ matrix.build.ARCH }} + arch: ${{ matrix.build.ARCH }} - name: Set the release version shell: bash @@ -115,8 +115,8 @@ jobs: fi echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV env: - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} - name: Setup node if: ${{ !contains(matrix.build.TARGET, 'musl') }} @@ -128,15 +128,15 @@ jobs: scope: "${{ vars.NPM_SCOPE }}" always-auth: true token: ${{ secrets.NPM_AUTH_TOKEN }} - + - name: Setup node for publishing if: ${{ !contains(matrix.build.TARGET, 'musl') }} working-directory: ./node run: | - npm config set registry https://registry.npmjs.org/ - npm config set '//registry.npmjs.org/:_authToken' ${{ secrets.NPM_AUTH_TOKEN }} - npm config set scope ${{ vars.NPM_SCOPE }} - + npm config set registry https://registry.npmjs.org/ + npm config set '//registry.npmjs.org/:_authToken' ${{ secrets.NPM_AUTH_TOKEN }} + npm config set scope ${{ vars.NPM_SCOPE }} + - name: Update package version in config.toml uses: ./.github/workflows/update-glide-version with: @@ -154,7 +154,7 @@ jobs: publish: "true" github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - + - name: Check if RC and set a distribution tag for the package shell: bash run: | @@ -171,9 +171,9 @@ jobs: - name: Check that the release version dont have typo init if: ${{ github.event_name != 'pull_request' && contains(env.RELEASE_VERSION, '-') && !contains(env.RELEASE_VERSION, 'rc') }} run: | - echo "The release version "${GITHUB_REF:11}" contains a typo, please fix it" - echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release." - exit 1 + echo "The release version "${GITHUB_REF:11}" contains a typo, please fix it" + echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release." + exit 1 - name: Publish to NPM if: github.event_name != 'pull_request' @@ -203,11 +203,16 @@ jobs: if: ${{ matrix.build.ARCH == 'arm64' }} shell: bash run: | +<<<<<<< HEAD echo "Resetting repository" git clean -xdf git reset --hard git fetch git checkout ${{ github.sha }} +======= + git reset --hard + git clean -xdf +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) publish-base-to-npm: if: github.event_name != 'pull_request' @@ -261,7 +266,7 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - + - name: Check if RC and set a distribution tag for the package shell: bash run: | @@ -274,7 +279,7 @@ jobs: export npm_tag="latest" fi echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV - + - name: Publish the base package if: github.event_name != 'pull_request' shell: bash @@ -299,7 +304,7 @@ jobs: strategy: fail-fast: false matrix: - build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} + build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} steps: - name: Setup self-hosted runner access if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }} @@ -308,20 +313,20 @@ jobs: - name: install Redis and git for alpine if: ${{ contains(matrix.build.TARGET, 'musl') }} run: | - apk update - apk add redis git - node -v - + apk update + apk add redis git + node -v + - name: install Redis and Python for ubuntu if: ${{ contains(matrix.build.TARGET, 'linux-gnu') }} run: | - sudo apt-get update - sudo apt-get install redis-server python3 + sudo apt-get update + sudo apt-get install redis-server python3 - name: install Redis, Python for macos if: ${{ contains(matrix.build.RUNNER, 'mac') }} run: | - brew install redis python3 + brew install redis python3 - name: Checkout if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl'}} @@ -384,5 +389,5 @@ jobs: if: ${{ contains(matrix.build.RUNNER, 'self-hosted') }} shell: bash run: | - git reset --hard - git clean -xdf + git reset --hard + git clean -xdf diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml index fcea61ee6b..2eff2a3f1a 100644 --- a/.github/workflows/ort.yml +++ b/.github/workflows/ort.yml @@ -1,34 +1,33 @@ - name: The OSS Review Toolkit (ORT) on: schedule: - - cron: "0 0 * * *" + - cron: "0 0 * * *" pull_request: - paths: - - .github/workflows/ort.yml - - .github/workflows/run-ort-tools/action.yml - - utils/get_licenses_from_ort.py + paths: + - .github/workflows/ort.yml + - .github/workflows/run-ort-tools/action.yml + - utils/get_licenses_from_ort.py workflow_dispatch: - inputs: - branch: - description: 'The branch to run against the ORT tool' - required: true - version: - description: 'The release version of GLIDE' - required: true + inputs: + branch: + description: "The branch to run against the ORT tool" + required: true + version: + description: "The release version of GLIDE" + required: true jobs: run-ort: if: github.repository_owner == 'valkey-io' name: Create attribution files runs-on: ubuntu-latest strategy: - fail-fast: false - env: - PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON" - NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" - RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST" - JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA" + fail-fast: false + env: + PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON" + NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" + RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST" + JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA" steps: - name: Set the release version shell: bash @@ -36,17 +35,17 @@ jobs: export version=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo '255.255.255'; else echo "$INPUT_VERSION"; fi` echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV env: - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} - + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} + - name: Set the base branch run: | - export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi` - echo "Base branch is: ${BASE_BRANCH}" - echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV + export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi` + echo "Base branch is: ${BASE_BRANCH}" + echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV env: - EVENT_NAME: ${{ github.event_name }} - INPUT_BRANCH: ${{ github.event.inputs.branch }} + EVENT_NAME: ${{ github.event_name }} + INPUT_BRANCH: ${{ github.event.inputs.branch }} - name: Checkout uses: actions/checkout@v4 @@ -64,73 +63,73 @@ jobs: uses: actions/cache@v4 id: cache-ort with: - path: | - ./ort - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-ort + path: | + ./ort + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-ort - name: Checkout ORT Repository if: steps.cache-ort.outputs.cache-hit != 'true' uses: actions/checkout@v4 - with: + with: repository: "oss-review-toolkit/ort" path: "./ort" ref: "26.0.0" submodules: recursive - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1.78 + uses: dtolnay/rust-toolchain@1.78 - name: Install ORT if: steps.cache-ort.outputs.cache-hit != 'true' working-directory: ./ort/ run: | - export JAVA_OPTS="$JAVA_OPTS -Xmx8g" - ./gradlew installDist + export JAVA_OPTS="$JAVA_OPTS -Xmx8g" + ./gradlew installDist - name: Create ORT config file run: | - mkdir -p ~/.ort/config - cat << EOF > ~/.ort/config/config.yml - ort: - analyzer: - allowDynamicVersions: true - enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector] - EOF - cat ~/.ort/config/config.yml + mkdir -p ~/.ort/config + cat << EOF > ~/.ort/config/config.yml + ort: + analyzer: + allowDynamicVersions: true + enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector] + EOF + cat ~/.ort/config/config.yml - ### NodeJS ### + ### NodeJS ### - name: Set up Node.js 16.x uses: actions/setup-node@v4 with: node-version: 16.x - - name: Create package.json file for the Node wrapper + - name: Create package.json file for the Node wrapper uses: ./.github/workflows/node-create-package-file with: - release_version: ${{ env.RELEASE_VERSION }} - os: "ubuntu-latest" + release_version: ${{ env.RELEASE_VERSION }} + os: "ubuntu-latest" - name: Fix Node base NPM package.json file for ORT working-directory: ./node/npm/glide run: | - # Remove the glide-rs dependency to avoid duplication - sed -i '/ "glide-rs":/d' ../../package.json - export pkg_name=valkey-glide-base - export package_version="${{ env.RELEASE_VERSION }}" - export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - mv package.json package.json.tmpl - envsubst < package.json.tmpl > "package.json" - cat package.json - + # Remove the glide-rs dependency to avoid duplication + sed -i '/ "glide-rs":/d' ../../package.json + export pkg_name=valkey-glide-base + export package_version="${{ env.RELEASE_VERSION }}" + export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` + mv package.json package.json.tmpl + envsubst < package.json.tmpl > "package.json" + cat package.json + - name: Run ORT tools for Node uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/node" - - ### Python ### + folder_path: "${{ github.workspace }}/node" + + ### Python ### - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -146,14 +145,14 @@ jobs: - name: Run ORT tools for Python uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/python" + folder_path: "${{ github.workspace }}/python" ### Rust ### - name: Run ORT tools for Rust uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/glide-core" + folder_path: "${{ github.workspace }}/glide-core" ### Java ### @@ -163,60 +162,60 @@ jobs: distribution: "temurin" java-version: 11 - - name: Run ORT tools for Java + - name: Run ORT tools for Java uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/java" + folder_path: "${{ github.workspace }}/java" ### Process results ### - name: Check for diff run: | - cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS - cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS - cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS - cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS - GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS` - if [ -n "$GIT_DIFF" ]; then - echo "FOUND_DIFF=true" >> $GITHUB_ENV - else - echo "FOUND_DIFF=false" >> $GITHUB_ENV - fi + cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS + cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS + cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS + cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS + GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS` + if [ -n "$GIT_DIFF" ]; then + echo "FOUND_DIFF=true" >> $GITHUB_ENV + else + echo "FOUND_DIFF=false" >> $GITHUB_ENV + fi - name: Retrieve licenses list working-directory: ./utils run: | - { - echo 'LICENSES_LIST<> "$GITHUB_ENV" + { + echo 'LICENSES_LIST<> "$GITHUB_ENV" ### Create PR ### - name: Create pull request if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }} run: | - export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi` - echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}" - git config --global user.email "valkey-glide@lists.valkey.io" - git config --global user.name "ort-bot" - git checkout -b ${BRANCH_NAME} - git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS - git commit -m "Updated attribution files" -s - git push --set-upstream origin ${BRANCH_NAME} -f - title="Updated attribution files for ${BRANCH_NAME}" - gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}' + export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi` + echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}" + git config --global user.email "valkey-glide@lists.valkey.io" + git config --global user.name "ort-bot" + git checkout -b ${BRANCH_NAME} + git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS + git commit -m "Updated attribution files" -s + git push --set-upstream origin ${BRANCH_NAME} -f + title="Updated attribution files for ${BRANCH_NAME}" + gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}' env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} - name: Get current date id: date run: | - CURR_DATE=$(date +'%Y-%m-%d-%H') - echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT + CURR_DATE=$(date +'%Y-%m-%d-%H') + echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT - name: Upload the final package list continue-on-error: true diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index 5d547ac7f5..8d4a0a499f 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -112,7 +112,18 @@ jobs: if: ${{ !contains(matrix.build.RUNNER, 'self-hosted') }} uses: actions/setup-python@v5 with: +<<<<<<< HEAD python-version: "3.12" +======= + python-version: "3.10" + + - name: Set up Python older versions for MacOS + if: startsWith(matrix.build.NAMED_OS, 'darwin') + run: | + brew update + brew install python@3.9 + +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Setup Python for self-hosted Ubuntu runners if: contains(matrix.build.RUNNER, 'self-hosted') diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 6c45d7707c..d89b3c64ec 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -37,57 +37,52 @@ on: - .github/json_matrices/engine-matrix.json - .github/workflows/start-self-hosted-runner/action.yml workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: false + default: "false" + + schedule: + - cron: "0 2 * * *" concurrency: group: python-${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output}} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - - test: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: python + dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + + test-python: runs-on: ${{ matrix.host.RUNNER }} - needs: load-engine-matrix + needs: get-matrices timeout-minutes: 35 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" - - "3.12" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} @@ -137,42 +132,31 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** - - test-pubsub: + test-pubsub-python: runs-on: ${{ matrix.host.RUNNER }} - needs: load-engine-matrix + needs: get-matrices timeout-minutes: 35 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" - - "3.12" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 +<<<<<<< HEAD +======= + with: + submodules: recursive + +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - + - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper with: @@ -180,14 +164,14 @@ jobs: target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} - + - name: Test pubsub with pytest working-directory: ./python run: | source .env/bin/activate cd python/tests/ pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html - + - name: Upload test reports if: always() continue-on-error: true @@ -235,8 +219,8 @@ jobs: working-directory: ./python run: | black --check --diff . - build-amazonlinux-latest: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') runs-on: ubuntu-latest container: amazonlinux:latest timeout-minutes: 15 @@ -282,6 +266,7 @@ jobs: name: smoke-test-report-amazon-linux path: | python/python/tests/pytest_report.html +<<<<<<< HEAD test-modules: if: (github.repository_owner == 'valkey-io' && github.event_name == 'workflow_dispatch') || github.event.pull_request.head.repo.owner.login == 'valkey-io' @@ -290,10 +275,41 @@ jobs: runs-on: [self-hosted, linux, ARM64] timeout-minutes: 15 +======= + + start-self-hosted-runner: + if: github.event.pull_request.head.repo.owner.login == 'valkey-io' + runs-on: ubuntu-latest + environment: AWS_ACTIONS + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start self hosted EC2 runner + uses: ./.github/workflows/start-self-hosted-runner + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + + test-modules: + needs: [start-self-hosted-runner, get-matrices] + name: Running Module Tests + runs-on: ${{ matrix.host.RUNNER }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) steps: - name: Setup self-hosted runner access if: ${{ contains(matrix.host.RUNNER, 'self-hosted') }} run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide +<<<<<<< HEAD - uses: actions/checkout@v4 @@ -304,6 +320,27 @@ jobs: target: aarch64-unknown-linux-gnu github-token: ${{ secrets.GITHUB_TOKEN }} +======= + + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python for self-hosted Ubuntu runners + run: | + sudo apt update -y + sudo apt upgrade -y + sudo apt install python3 python3-venv python3-pip -y + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Test with pytest working-directory: ./python run: | @@ -316,6 +353,10 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: +<<<<<<< HEAD name: modules-test-report +======= + name: modules-test-report-${{ matrix.host.TARGET }}-python-${{ matrix.python }}-server-${{ matrix.engine.version }} +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) path: | python/python/tests/pytest_report.html diff --git a/.github/workflows/run-ort-tools/action.yml b/.github/workflows/run-ort-tools/action.yml index 5686f21b58..2b55517700 100644 --- a/.github/workflows/run-ort-tools/action.yml +++ b/.github/workflows/run-ort-tools/action.yml @@ -13,11 +13,11 @@ runs: working-directory: ./ort/ shell: bash run: | - echo "Running ORT tools for ${{ inputs.folder_path }}" - FOLDER=${{ inputs.folder_path }} - mkdir $FOLDER/ort_results - # Analyzer (analyzer-result.json) - ./gradlew cli:run --args="analyze -i $FOLDER -o $FOLDER/ort_results -f JSON" - - # NOTICE DEFAULT - ./gradlew cli:run --args="report -i $FOLDER/ort_results/analyzer-result.json -o $FOLDER/ort_results/ -f PlainTextTemplate" + echo "Running ORT tools for ${{ inputs.folder_path }}" + FOLDER=${{ inputs.folder_path }} + mkdir $FOLDER/ort_results + # Analyzer (analyzer-result.json) + ./gradlew cli:run --args="analyze -i $FOLDER -o $FOLDER/ort_results -f JSON" + + # NOTICE DEFAULT + ./gradlew cli:run --args="report -i $FOLDER/ort_results/analyzer-result.json -o $FOLDER/ort_results/ -f PlainTextTemplate" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2fdfa77c1f..5aec66ec70 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -40,17 +40,26 @@ env: jobs: load-engine-matrix: +<<<<<<< HEAD runs-on: ubuntu-latest outputs: matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT +======= + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT build: runs-on: ubuntu-latest diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index a9cf3db6df..3a0e524d06 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -2,7 +2,11 @@ name: Semgrep on: # Scan changed files in PRs (diff-aware scanning): +<<<<<<< HEAD pull_request: +======= + pull_request: {} +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) # Scan on-demand through GitHub Actions interface: workflow_dispatch: inputs: @@ -10,10 +14,14 @@ on: description: "The branch to run against the semgrep tool" required: true push: +<<<<<<< HEAD branches: - main - release-* - v* +======= + branches: ["main"] +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) # Schedule the CI job (this method uses cron syntax): schedule: - cron: "0 8 * * *" # Sets Semgrep to scan every day at 08:00 UTC. @@ -34,6 +42,12 @@ jobs: steps: # Fetch project source with GitHub Actions Checkout. +<<<<<<< HEAD - uses: actions/checkout@v4 # Run the "semgrep ci" command on the command line of the docker image. - run: semgrep ci --config auto --no-suppress-errors --exclude-rule generic.secrets.security.detected-private-key.detected-private-key +======= + - uses: actions/checkout@v3 + # Run the "semgrep ci" command on the command line of the docker image. + - run: semgrep ci --config auto --no-suppress-errors +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) diff --git a/.github/workflows/setup-musl-on-linux/action.yml b/.github/workflows/setup-musl-on-linux/action.yml index f270c27507..cbdcdee2f1 100644 --- a/.github/workflows/setup-musl-on-linux/action.yml +++ b/.github/workflows/setup-musl-on-linux/action.yml @@ -30,28 +30,37 @@ runs: run: | apk update apk add bash git sed python3 - + - name: Skip all steps if not on ARM64 shell: bash if: ${{ inputs.arch != 'arm64' }} run: exit 0 - # Currently "Checkout" action is not supported for musl on ARM64, so the checkout is happening on the runner and + # Currently "Checkout" action is not supported for musl on ARM64, so the checkout is happening on the runner and # here we just making sure we getting the clean repo - name: Clean repository for musl on ARM64 shell: bash run: | +<<<<<<< HEAD git config --global --add safe.directory "${{ inputs.workspace }}" git fetch origin ${{ github.sha }} git checkout ${{ github.sha }} git clean -xdf git reset --hard +======= + git config --global --add safe.directory "${{ inputs.workspace }}" + git clean -xdf + git reset --hard + git submodule sync + git submodule update --init --recursive + +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Set up access for musl on ARM shell: bash run: | chown -R $(whoami):$(whoami) $GITHUB_WORKSPACE - + - name: Setup node shell: bash working-directory: ./node diff --git a/.github/workflows/start-self-hosted-runner/action.yml b/.github/workflows/start-self-hosted-runner/action.yml index 45038b2d1d..e929e1d2d4 100644 --- a/.github/workflows/start-self-hosted-runner/action.yml +++ b/.github/workflows/start-self-hosted-runner/action.yml @@ -19,14 +19,14 @@ runs: with: role-to-assume: ${{ inputs.role-to-assume }} aws-region: ${{ inputs.aws-region }} - + - name: Start EC2 self hosted runner shell: bash run: | sudo snap refresh sudo snap install aws-cli --classic command_id=$(aws ssm send-command --instance-ids ${{ inputs.ec2-instance-id }} --document-name StartGithubSelfHostedRunner --query Command.CommandId --output text) - + while [[ "$invoke_status" != "Success" && "$invoke_status" != "Failed" ]]; do invoke_status=$(aws ssm list-command-invocations --command-id $command_id --query 'CommandInvocations[0].Status' --output text) echo "Current Status: $invoke_status" @@ -41,4 +41,4 @@ runs: fi done - echo "Final Command Status: $invoke_status" + echo "Final Command Status: $invoke_status" diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..dd449725e1 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +*.md diff --git a/.vscode/settings.json b/.vscode/settings.json index 72bcb0d6f7..aca9b74cf0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,7 +27,7 @@ "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "[github-actions-workflow]": { - "editor.defaultFormatter": "redhat.vscode-yaml" + "editor.defaultFormatter": "esbenp.prettier-vscode" }, "yaml.schemas": { "https://json.schemastore.org/github-workflow.json": [ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d809b3801..9da17328a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ #### Changes +<<<<<<< HEAD * Python: Python: FT.PROFILE command added ([#2543](https://github.com/valkey-io/valkey-glide/pull/2543)) * Python: Python: FT.AGGREGATE command added([#2530](https://github.com/valkey-io/valkey-glide/pull/2530)) * Python: Add JSON.OBJLEN command ([#2495](https://github.com/valkey-io/valkey-glide/pull/2495)) @@ -58,6 +59,9 @@ * Node: Add `JSON.STRLEN` and `JSON.STRAPPEND` command ([#2537](https://github.com/valkey-io/valkey-glide/pull/2537)) * Node: Add `FT.SEARCH` ([#2551](https://github.com/valkey-io/valkey-glide/pull/2551)) * Python: Fix example ([#2556](https://github.com/valkey-io/valkey-glide/issues/2556)) +======= +* Python: Add JSON.ARRLEN command ([#2403](https://github.com/valkey-io/valkey-glide/pull/2403)) +>>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) #### Breaking Changes diff --git a/node/tests/GlideClient.test.ts b/node/tests/GlideClient.test.ts index e0ac65e169..7e0a75ddec 100644 --- a/node/tests/GlideClient.test.ts +++ b/node/tests/GlideClient.test.ts @@ -40,7 +40,6 @@ import { flushAndCloseClient, generateLuaLibCode, getClientConfigurationOption, - getServerVersion, parseCommandLineArgs, parseEndpoints, transactionTest, @@ -59,12 +58,10 @@ describe("GlideClient", () => { const standaloneAddresses = parseCommandLineArgs()["standalone-endpoints"]; cluster = standaloneAddresses - ? await ValkeyCluster.initFromExistingCluster( - false, + ? await RedisCluster.initFromExistingCluster( parseEndpoints(standaloneAddresses), - getServerVersion, ) - : await ValkeyCluster.createCluster(false, 1, 1, getServerVersion); + : await RedisCluster.createCluster(false, 1, 1); }, 20000); afterEach(async () => { @@ -135,6 +132,47 @@ describe("GlideClient", () => { }, ); + it.each([ProtocolVersion.RESP2, ProtocolVersion.RESP3])( + "check that blocking commands returns never timeout_%p", + async (protocol) => { + client = await GlideClient.createClient( + getClientConfigurationOption( + cluster.getAddresses(), + protocol, + 300, + ), + ); + + const promiseList = [ + client.blmove( + "source", + "destination", + ListDirection.LEFT, + ListDirection.LEFT, + 0.1, + ), + client.bzpopmax(["key1", "key2"], 0), + client.bzpopmin(["key1", "key2"], 0), + ]; + + try { + for (const promise of promiseList) { + const timeoutPromise = new Promise((resolve) => { + setTimeout(resolve, 500); + }); + await Promise.race([promise, timeoutPromise]); + } + } finally { + for (const promise of promiseList) { + await Promise.resolve([promise]); + } + + client.close(); + } + }, + 5000, + ); + it.each([ProtocolVersion.RESP2, ProtocolVersion.RESP3])( "select dbsize flushdb test %p", async (protocol) => { diff --git a/node/tests/GlideClusterClient.test.ts b/node/tests/GlideClusterClient.test.ts index 6f29f99884..eea642aef5 100644 --- a/node/tests/GlideClusterClient.test.ts +++ b/node/tests/GlideClusterClient.test.ts @@ -47,7 +47,6 @@ import { generateLuaLibCode, getClientConfigurationOption, getFirstResult, - getServerVersion, intoArray, intoString, parseCommandLineArgs, @@ -68,13 +67,17 @@ describe("GlideClusterClient", () => { const clusterAddresses = parseCommandLineArgs()["cluster-endpoints"]; // Connect to cluster or create a new one based on the parsed addresses cluster = clusterAddresses - ? await ValkeyCluster.initFromExistingCluster( - true, + ? (await RedisCluster.initFromExistingCluster( parseEndpoints(clusterAddresses), - getServerVersion, - ) - : // setting replicaCount to 1 to facilitate tests routed to replicas - await ValkeyCluster.createCluster(true, 3, 1, getServerVersion); + )) + ? await ValkeyCluster.initFromExistingCluster( + true, + parseEndpoints(clusterAddresses), + getServerVersion, + ) + : // setting replicaCount to 1 to facilitate tests routed to replicas + await RedisCluster.createCluster(true, 3, 1) + : await ValkeyCluster.createCluster(true, 3, 1, getServerVersion); }, 20000); afterEach(async () => { @@ -370,6 +373,16 @@ describe("GlideClusterClient", () => { const client = await GlideClusterClient.createClient( getClientConfigurationOption(cluster.getAddresses(), protocol), ); + const lmpopArr = []; + + if (!cluster.checkIfServerVersionLessThan("7.0.0")) { + lmpopArr.push( + client.lmpop(["abc", "def"], ListDirection.LEFT, 1), + ); + lmpopArr.push( + client.blmpop(["abc", "def"], ListDirection.RIGHT, 0.1, 1), + ); + } const promises: Promise[] = [ client.blpop(["abc", "zxy", "lkn"], 0.1), @@ -391,10 +404,7 @@ describe("GlideClusterClient", () => { client.sdiffstore("abc", ["zxy", "lkn"]), client.sortStore("abc", "zyx"), client.sortStore("abc", "zyx", { isAlpha: true }), - client.lmpop(["abc", "def"], ListDirection.LEFT, { count: 1 }), - client.blmpop(["abc", "def"], ListDirection.RIGHT, 0.1, { - count: 1, - }), + ...lmpopArr, client.bzpopmax(["abc", "def"], 0.5), client.bzpopmin(["abc", "def"], 0.5), client.xread({ abc: "0-0", zxy: "0-0", lkn: "0-0" }), diff --git a/python/python/glide/async_commands/server_modules/json.py b/python/python/glide/async_commands/server_modules/json.py index 3c98672d9f..987d1a0094 100644 --- a/python/python/glide/async_commands/server_modules/json.py +++ b/python/python/glide/async_commands/server_modules/json.py @@ -704,6 +704,60 @@ async def debug_memory( ) +async def arrlen( + client: TGlideClient, + key: TEncodable, + path: Optional[TEncodable] = None, +) -> Optional[TJsonResponse[int]]: + """ + Retrieves the length of the array at the specified `path` within the JSON document stored at `key`. + + Args: + client (TGlideClient): The client to execute the command. + key (TEncodable): The key of the JSON document. + path (Optional[TEncodable]): The path within the JSON document. Defaults to None. + + Returns: + Optional[TJsonResponse[int]]: + For JSONPath (`path` starts with `$`): + Returns a list of integer replies for every possible path, indicating the length of the array, + or None for JSON values matching the path that are not an array. + If `path` doesn't exist, an empty array will be returned. + For legacy path (`path` doesn't starts with `$`): + Returns the length of the array at `path`. + If multiple paths match, the length of the first array match is returned. + If the JSON value at `path` is not a array or if `path` doesn't exist, an error is raised. + If `key` doesn't exist, None is returned. + + Examples: + >>> from glide import json + >>> await json.set(client, "doc", "$", '{"a": [1, 2, 3], "b": {"a": [1, 2], "c": {"a": 42}}}') + b'OK' # JSON is successfully set for doc + >>> await json.arrlen(client, "doc", "$") + [None] # No array at the root path. + >>> await json.arrlen(client, "doc", "$.a") + [3] # Retrieves the length of the array at path $.a. + >>> await json.arrlen(client, "doc", "$..a") + [3, 2, None] # Retrieves lengths of arrays found at all levels of the path `..a`. + >>> await json.arrlen(client, "doc", "..a") + 3 # Legacy path retrieves the first array match at path `..a`. + >>> await json.arrlen(client, "non_existing_key", "$.a") + None # Returns None because the key does not exist. + + >>> await json.set(client, "doc", "$", '[1, 2, 3, 4]') + b'OK' # JSON is successfully set for doc + >>> await json.arrlen(client, "doc") + 4 # Retrieves lengths of arrays in root. + """ + args = ["JSON.ARRLEN", key] + if path: + args.append(path) + return cast( + Optional[TJsonResponse[int]], + await client.custom_command(args), + ) + + async def delete( client: TGlideClient, key: TEncodable, From e801c03f9ef47d81a177d04c7b0f2e2875b5b1a4 Mon Sep 17 00:00:00 2001 From: avifenesh Date: Sun, 3 Nov 2024 20:42:15 +0000 Subject: [PATCH 3/8] CI - Minimal and full CI matrix impl (#2051) * CI - Minimal and full CI matrix impl Signed-off-by: avifenesh * Fix mypy failing (#2453) --------- Signed-off-by: Shoham Elias Signed-off-by: avifenesh * Python: adds JSON.ARRLEN command (#2403) --------- Signed-off-by: Shoham Elias Signed-off-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> Signed-off-by: avifenesh --------- Signed-off-by: avifenesh Signed-off-by: Shoham Elias Signed-off-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> Co-authored-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> --- .github/json_matrices/build-matrix.json | 31 --------- .github/workflows/codeql.yml | 12 ---- .github/workflows/go.yml | 6 -- .../install-shared-dependencies/action.yml | 6 -- .github/workflows/node.yml | 26 ------- .github/workflows/npm-cd.yml | 13 ++-- .github/workflows/pypi-cd.yml | 11 --- .github/workflows/python.yml | 68 +------------------ .github/workflows/rust.yml | 11 --- .github/workflows/semgrep.yml | 14 ---- .../workflows/setup-musl-on-linux/action.yml | 13 +--- CHANGELOG.md | 4 -- 12 files changed, 7 insertions(+), 208 deletions(-) diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index e91aeeefe3..fc02093b9f 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -5,16 +5,11 @@ "RUNNER": "ubuntu-latest", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", -<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ] -======= - "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["always", "python", "node", "java"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "ubuntu", @@ -26,18 +21,12 @@ ], "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-gnu", -<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ], "CONTAINER": "2_28" -======= - "PACKAGE_MANAGERS": ["pypi", "npm"], - "CONTAINER": "2_28", - "run": ["python", "node", "java"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "macos", @@ -45,16 +34,11 @@ "RUNNER": "macos-12", "ARCH": "x64", "TARGET": "x86_64-apple-darwin", -<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ] -======= - "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["python", "node", "java"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "macos", @@ -62,16 +46,11 @@ "RUNNER": "macos-latest", "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", -<<<<<<< HEAD "PACKAGE_MANAGERS": [ "pypi", "npm", "maven" ] -======= - "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["python", "node", "java"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "ubuntu", @@ -85,14 +64,9 @@ ], "IMAGE": "node:lts-alpine3.19", "CONTAINER_OPTIONS": "--user root --privileged --rm", -<<<<<<< HEAD "PACKAGE_MANAGERS": [ "npm" ] -======= - "PACKAGE_MANAGERS": ["npm"], - "run": ["node"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) }, { "OS": "ubuntu", @@ -102,13 +76,8 @@ "RUNNER": "ubuntu-latest", "IMAGE": "node:lts-alpine3.19", "CONTAINER_OPTIONS": "--user root --privileged", -<<<<<<< HEAD "PACKAGE_MANAGERS": [ "npm" ] -======= - "PACKAGE_MANAGERS": ["npm"], - "run": ["node"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) } ] diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 37ccbf1c2f..fde89563bf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,33 +2,21 @@ name: "CodeQL" on: push: -<<<<<<< HEAD - branches: -======= branches: ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - "main" - "v.?[0-9]+.[0-9]+.[0-9]+" - "v.?[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+" -<<<<<<< HEAD - release-* - pull_request: - branches: -======= pull_request: branches: ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - "main" - "v.?[0-9]+.[0-9]+.[0-9]+" - "v.?[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+.[0-9]+" - "v?[0-9]+.[0-9]+" -<<<<<<< HEAD - release-* -======= ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) schedule: - cron: "37 18 * * 6" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 95373ffc09..40d7c77cd7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,14 +2,10 @@ name: Go CI on: push: -<<<<<<< HEAD branches: - main - release-* - v* -======= - branches: ["main"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) paths: - glide-core/src/** - glide-core/redis-rs/redis/src/** @@ -77,7 +73,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Go ${{ matrix.go }} uses: actions/setup-go@v5 @@ -207,7 +202,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/workflows/lint-rust with: diff --git a/.github/workflows/install-shared-dependencies/action.yml b/.github/workflows/install-shared-dependencies/action.yml index ac75e48596..49bc44cb24 100644 --- a/.github/workflows/install-shared-dependencies/action.yml +++ b/.github/workflows/install-shared-dependencies/action.yml @@ -23,14 +23,8 @@ inputs: - x86_64-unknown-linux-musl engine-version: description: "Engine version to install" -<<<<<<< HEAD required: false type: string -======= - required: true - type: string - ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) github-token: description: "GITHUB_TOKEN, GitHub App installation access token" required: true diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index e02ef98806..8f3988120a 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -51,24 +51,7 @@ env: CARGO_TERM_COLOR: always jobs: -<<<<<<< HEAD - load-engine-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - - test-ubuntu-latest: -======= get-matrices: ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) runs-on: ubuntu-latest # Avoid running on schedule for forks if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') @@ -97,17 +80,11 @@ jobs: node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output)}} steps: - uses: actions/checkout@v4 - -<<<<<<< HEAD - - name: Use Node.js 16.x - uses: actions/setup-node@v4 -======= - name: Setup Node uses: actions/setup-node@v4 env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) with: node-version: ${{ matrix.node }} @@ -161,7 +138,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: lint node rust uses: ./.github/workflows/lint-rust @@ -232,7 +208,6 @@ jobs: apk add git - uses: actions/checkout@v4 - - name: Setup musl on Linux uses: ./.github/workflows/setup-musl-on-linux @@ -278,7 +253,6 @@ jobs: run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - uses: actions/checkout@v4 - - name: Use Node.js 16.x uses: actions/setup-node@v4 diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 83011739b4..e2c7b42ff3 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -203,16 +203,11 @@ jobs: if: ${{ matrix.build.ARCH == 'arm64' }} shell: bash run: | -<<<<<<< HEAD - echo "Resetting repository" - git clean -xdf - git reset --hard - git fetch - git checkout ${{ github.sha }} -======= - git reset --hard + echo "Resetting repository" git clean -xdf ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) + git reset --hard + git fetch + git checkout ${{ github.sha }} publish-base-to-npm: if: github.event_name != 'pull_request' diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index 8d4a0a499f..5d547ac7f5 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -112,18 +112,7 @@ jobs: if: ${{ !contains(matrix.build.RUNNER, 'self-hosted') }} uses: actions/setup-python@v5 with: -<<<<<<< HEAD python-version: "3.12" -======= - python-version: "3.10" - - - name: Set up Python older versions for MacOS - if: startsWith(matrix.build.NAMED_OS, 'darwin') - run: | - brew update - brew install python@3.9 - ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Setup Python for self-hosted Ubuntu runners if: contains(matrix.build.RUNNER, 'self-hosted') diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d89b3c64ec..abf563b98a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -79,7 +79,6 @@ jobs: host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 - - name: Set up Python uses: actions/setup-python@v5 @@ -144,14 +143,7 @@ jobs: host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 -<<<<<<< HEAD - - -======= - with: - submodules: recursive ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Set up Python uses: actions/setup-python@v5 with: @@ -186,7 +178,6 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@v4 - - name: lint rust uses: ./.github/workflows/lint-rust @@ -266,50 +257,18 @@ jobs: name: smoke-test-report-amazon-linux path: | python/python/tests/pytest_report.html -<<<<<<< HEAD - + test-modules: if: (github.repository_owner == 'valkey-io' && github.event_name == 'workflow_dispatch') || github.event.pull_request.head.repo.owner.login == 'valkey-io' environment: AWS_ACTIONS name: Running Module Tests runs-on: [self-hosted, linux, ARM64] timeout-minutes: 15 - -======= - start-self-hosted-runner: - if: github.event.pull_request.head.repo.owner.login == 'valkey-io' - runs-on: ubuntu-latest - environment: AWS_ACTIONS - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Start self hosted EC2 runner - uses: ./.github/workflows/start-self-hosted-runner - with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} - - test-modules: - needs: [start-self-hosted-runner, get-matrices] - name: Running Module Tests - runs-on: ${{ matrix.host.RUNNER }} - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} - python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} - ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) steps: - name: Setup self-hosted runner access if: ${{ contains(matrix.host.RUNNER, 'self-hosted') }} run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide -<<<<<<< HEAD - uses: actions/checkout@v4 @@ -320,27 +279,6 @@ jobs: target: aarch64-unknown-linux-gnu github-token: ${{ secrets.GITHUB_TOKEN }} -======= - - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Python for self-hosted Ubuntu runners - run: | - sudo apt update -y - sudo apt upgrade -y - sudo apt install python3 python3-venv python3-pip -y - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Test with pytest working-directory: ./python run: | @@ -353,10 +291,6 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: -<<<<<<< HEAD name: modules-test-report -======= - name: modules-test-report-${{ matrix.host.TARGET }}-python-${{ matrix.python }}-server-${{ matrix.engine.version }} ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) path: | python/python/tests/pytest_report.html diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5aec66ec70..02274c06e0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -40,21 +40,12 @@ env: jobs: load-engine-matrix: -<<<<<<< HEAD - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 -======= runs-on: ubuntu-latest outputs: matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Load the engine matrix id: load-engine-matrix @@ -72,7 +63,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies @@ -107,7 +97,6 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v4 - - uses: ./.github/workflows/lint-rust with: diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 3a0e524d06..10523666fa 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -2,11 +2,7 @@ name: Semgrep on: # Scan changed files in PRs (diff-aware scanning): -<<<<<<< HEAD - pull_request: -======= pull_request: {} ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) # Scan on-demand through GitHub Actions interface: workflow_dispatch: inputs: @@ -14,14 +10,10 @@ on: description: "The branch to run against the semgrep tool" required: true push: -<<<<<<< HEAD branches: - main - release-* - v* -======= - branches: ["main"] ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) # Schedule the CI job (this method uses cron syntax): schedule: - cron: "0 8 * * *" # Sets Semgrep to scan every day at 08:00 UTC. @@ -42,12 +34,6 @@ jobs: steps: # Fetch project source with GitHub Actions Checkout. -<<<<<<< HEAD - uses: actions/checkout@v4 # Run the "semgrep ci" command on the command line of the docker image. - run: semgrep ci --config auto --no-suppress-errors --exclude-rule generic.secrets.security.detected-private-key.detected-private-key -======= - - uses: actions/checkout@v3 - # Run the "semgrep ci" command on the command line of the docker image. - - run: semgrep ci --config auto --no-suppress-errors ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) diff --git a/.github/workflows/setup-musl-on-linux/action.yml b/.github/workflows/setup-musl-on-linux/action.yml index cbdcdee2f1..5d69f4063d 100644 --- a/.github/workflows/setup-musl-on-linux/action.yml +++ b/.github/workflows/setup-musl-on-linux/action.yml @@ -41,21 +41,12 @@ runs: - name: Clean repository for musl on ARM64 shell: bash run: | -<<<<<<< HEAD - git config --global --add safe.directory "${{ inputs.workspace }}" - git fetch origin ${{ github.sha }} - git checkout ${{ github.sha }} - git clean -xdf - git reset --hard - -======= git config --global --add safe.directory "${{ inputs.workspace }}" + git fetch origin ${{ github.sha }} + git checkout ${{ github.sha }} git clean -xdf git reset --hard - git submodule sync - git submodule update --init --recursive ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) - name: Set up access for musl on ARM shell: bash run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9da17328a4..4d809b3801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,4 @@ #### Changes -<<<<<<< HEAD * Python: Python: FT.PROFILE command added ([#2543](https://github.com/valkey-io/valkey-glide/pull/2543)) * Python: Python: FT.AGGREGATE command added([#2530](https://github.com/valkey-io/valkey-glide/pull/2530)) * Python: Add JSON.OBJLEN command ([#2495](https://github.com/valkey-io/valkey-glide/pull/2495)) @@ -59,9 +58,6 @@ * Node: Add `JSON.STRLEN` and `JSON.STRAPPEND` command ([#2537](https://github.com/valkey-io/valkey-glide/pull/2537)) * Node: Add `FT.SEARCH` ([#2551](https://github.com/valkey-io/valkey-glide/pull/2551)) * Python: Fix example ([#2556](https://github.com/valkey-io/valkey-glide/issues/2556)) -======= -* Python: Add JSON.ARRLEN command ([#2403](https://github.com/valkey-io/valkey-glide/pull/2403)) ->>>>>>> bcc6b00b (CI - Minimal and full CI matrix impl (#2051)) #### Breaking Changes From 7a05d9776c57c3ab70789b8620ddbf3d5b1d02fe Mon Sep 17 00:00:00 2001 From: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:44:16 +0300 Subject: [PATCH 4/8] CI - Minimal and full CI matrix impl (#2051) * CI - Minimal and full CI matrix impl Signed-off-by: avifenesh * Fix mypy failing (#2453) --------- Signed-off-by: Shoham Elias Signed-off-by: avifenesh * Python: adds JSON.ARRLEN command (#2403) --------- Signed-off-by: Shoham Elias Signed-off-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> Signed-off-by: avifenesh --------- Signed-off-by: avifenesh Signed-off-by: Shoham Elias Signed-off-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> Co-authored-by: Shoham Elias <116083498+shohamazon@users.noreply.github.com> --- .github/json_matrices/build-matrix.json | 41 ++++++++++---- .../async_commands/server_modules/json.py | 54 +++++++++++++++++++ 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index fc02093b9f..4920799ba5 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -7,8 +7,13 @@ "TARGET": "x86_64-unknown-linux-gnu", "PACKAGE_MANAGERS": [ "pypi", - "npm", - "maven" + "npm" + ], + "run": [ + "always", + "python", + "node", + "java" ] }, { @@ -23,10 +28,14 @@ "TARGET": "aarch64-unknown-linux-gnu", "PACKAGE_MANAGERS": [ "pypi", - "npm", - "maven" + "npm" ], - "CONTAINER": "2_28" + "CONTAINER": "2_28", + "run": [ + "python", + "node", + "java" + ] }, { "OS": "macos", @@ -36,8 +45,12 @@ "TARGET": "x86_64-apple-darwin", "PACKAGE_MANAGERS": [ "pypi", - "npm", - "maven" + "npm" + ], + "run": [ + "python", + "node", + "java" ] }, { @@ -48,8 +61,12 @@ "TARGET": "aarch64-apple-darwin", "PACKAGE_MANAGERS": [ "pypi", - "npm", - "maven" + "npm" + ], + "run": [ + "python", + "node", + "java" ] }, { @@ -66,6 +83,9 @@ "CONTAINER_OPTIONS": "--user root --privileged --rm", "PACKAGE_MANAGERS": [ "npm" + ], + "run": [ + "node" ] }, { @@ -78,6 +98,9 @@ "CONTAINER_OPTIONS": "--user root --privileged", "PACKAGE_MANAGERS": [ "npm" + ], + "run": [ + "node" ] } ] diff --git a/python/python/glide/async_commands/server_modules/json.py b/python/python/glide/async_commands/server_modules/json.py index 987d1a0094..e4bff32b79 100644 --- a/python/python/glide/async_commands/server_modules/json.py +++ b/python/python/glide/async_commands/server_modules/json.py @@ -758,6 +758,60 @@ async def arrlen( ) +async def arrlen( + client: TGlideClient, + key: TEncodable, + path: Optional[TEncodable] = None, +) -> Optional[TJsonResponse[int]]: + """ + Retrieves the length of the array at the specified `path` within the JSON document stored at `key`. + + Args: + client (TGlideClient): The client to execute the command. + key (TEncodable): The key of the JSON document. + path (Optional[TEncodable]): The path within the JSON document. Defaults to None. + + Returns: + Optional[TJsonResponse[int]]: + For JSONPath (`path` starts with `$`): + Returns a list of integer replies for every possible path, indicating the length of the array, + or None for JSON values matching the path that are not an array. + If `path` doesn't exist, an empty array will be returned. + For legacy path (`path` doesn't starts with `$`): + Returns the length of the array at `path`. + If multiple paths match, the length of the first array match is returned. + If the JSON value at `path` is not a array or if `path` doesn't exist, an error is raised. + If `key` doesn't exist, None is returned. + + Examples: + >>> from glide import json + >>> await json.set(client, "doc", "$", '{"a": [1, 2, 3], "b": {"a": [1, 2], "c": {"a": 42}}}') + b'OK' # JSON is successfully set for doc + >>> await json.arrlen(client, "doc", "$") + [None] # No array at the root path. + >>> await json.arrlen(client, "doc", "$.a") + [3] # Retrieves the length of the array at path $.a. + >>> await json.arrlen(client, "doc", "$..a") + [3, 2, None] # Retrieves lengths of arrays found at all levels of the path `..a`. + >>> await json.arrlen(client, "doc", "..a") + 3 # Legacy path retrieves the first array match at path `..a`. + >>> await json.arrlen(client, "non_existing_key", "$.a") + None # Returns None because the key does not exist. + + >>> await json.set(client, "doc", "$", '[1, 2, 3, 4]') + b'OK' # JSON is successfully set for doc + >>> await json.arrlen(client, "doc") + 4 # Retrieves lengths of arrays in root. + """ + args = ["JSON.ARRLEN", key] + if path: + args.append(path) + return cast( + Optional[TJsonResponse[int]], + await client.custom_command(args), + ) + + async def delete( client: TGlideClient, key: TEncodable, From 632b6c3c08d1166c2fb09467d7834827f6f1b767 Mon Sep 17 00:00:00 2001 From: Avi Fenesh <55848801+avifenesh@users.noreply.github.com> Date: Wed, 30 Oct 2024 20:36:20 +0200 Subject: [PATCH 5/8] Fix for - Minimal and full CI matrix impl #2051 (#2500) * Refactor tests to use async cleanup and improve error handling Signed-off-by: avifenesh * Enhance Jest configuration and add test setup file; update build scripts and dependencies Signed-off-by: avifenesh * Update devDependencies in package.json for hybrid-node-tests to latest versions Signed-off-by: avifenesh * Enhance test utilities and command tests with improved wait logic and version checks Signed-off-by: avifenesh * Refactor tests to assert expected replica reads are less than or equal to actual reads; update connection handling in utilities and allow unused imports in types Signed-off-by: avifenesh * Update dependencies and enhance PyO3 bindings; add new features and improve type handling Signed-off-by: avifenesh * Update GitHub workflows: enhance linting configurations, adjust engine version requirements, and remove obsolete Redis installation workflow Signed-off-by: avifenesh --------- Signed-off-by: avifenesh --- .github/DEVELOPER.md | 56 ++-- .github/json_matrices/build-matrix.json | 58 ++-- .github/json_matrices/engine-matrix.json | 4 +- .../supported-languages-versions.json | 16 +- .../workflows/build-node-wrapper/action.yml | 4 +- .../workflows/build-python-wrapper/action.yml | 2 +- .../workflows/create-test-matrices/action.yml | 62 +++-- .github/workflows/csharp.yml | 169 ++++++++---- .github/workflows/go.yml | 220 ++++++++------- .../action.yml | 43 ++- .github/workflows/install-redis/action.yml | 46 ---- .../install-shared-dependencies/action.yml | 6 +- .github/workflows/java-cd.yml | 2 +- .github/workflows/java.yml | 104 +++++--- .github/workflows/lint-rust/action.yml | 1 - .github/workflows/nightly.yml | 88 ++++++ .github/workflows/node.yml | 157 ++++++----- .github/workflows/npm-cd.yml | 11 +- .github/workflows/pypi-cd.yml | 28 +- .github/workflows/python.yml | 127 +++++---- .github/workflows/rust.yml | 59 ++-- glide-core/src/client/types.rs | 2 + glide-core/tests/test_standalone_client.rs | 2 +- glide-core/tests/utilities/mod.rs | 2 +- .../commonjs-test/package.json | 6 +- .../ecmascript-test/package.json | 6 +- node/jest.config.js | 5 +- node/package.json | 6 +- node/tests/AsyncClient.test.ts | 4 +- node/tests/GlideClient.test.ts | 51 ++-- node/tests/GlideClientInternals.test.ts | 7 +- node/tests/GlideClusterClient.test.ts | 41 +-- node/tests/PubSub.test.ts | 30 ++- node/tests/ScanTest.test.ts | 6 +- node/tests/SharedTests.ts | 251 ++++++++---------- node/tests/TestUtilities.ts | 127 +++------ node/tests/setup.js | 7 + python/Cargo.toml | 15 +- python/src/lib.rs | 35 +-- utils/TestUtils.ts | 58 ++-- 40 files changed, 1100 insertions(+), 824 deletions(-) rename .github/workflows/{install-valkey => install-engine}/action.yml (60%) delete mode 100644 .github/workflows/install-redis/action.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 node/tests/setup.js diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md index 710bd08dca..2acc4ccb68 100644 --- a/.github/DEVELOPER.md +++ b/.github/DEVELOPER.md @@ -1,17 +1,25 @@ # CI/CD Workflow Guide -TODO: Add a description of the CI/CD workflow and its components. - ### Overview Our CI/CD pipeline tests and builds our project across multiple languages, versions, and environments. This guide outlines the key components and processes of our workflow. ### Workflow Triggers -- Push to `main` branch - Pull requests -- Scheduled runs (daily) -- Manual trigger (workflow_dispatch) +- Pushes to `main` or release branches (PR merges) +- Scheduled runs (daily) - starts CI pipelines for all clients +- Manual trigger (`workflow_dispatch`) - a developer can start a client's pipeline or the scheduled one to run all pipelines on demand + +Job triggers + +### Test coverage + +There are two levels of testing: the basic one and full (_aka_ `full-matrix`). +Basic amount of testing is executed on every open and merged PR. The full set of tests is executed by the scheduled job. +A developer can select the level when starting a job, either scheduled or client's pipeline. + +Matrices ### Language-Specific Workflows @@ -24,9 +32,9 @@ Each language has its own workflow file with similar structure but language-spec While workflows are language-specific, the matrix files are shared across all workflows. Workflows are starting by loading the matrix files from the `.github/json_matrices` directory. -- `engine-matrix.json`: Defines the versions of Valkey engine to test against. +- `engine-matrix.json`: Defines the versions of the engine to test against. - `build-matrix.json`: Defines the host environments for testing. -- `supported-languages-version.json`: Defines the supported versions of languages. +- `supported-languages-versions.json`: Defines the supported versions of languages. All matrices have a `run` like field which specifies if the configuration should be tested on every workflow run. This allows for flexible control over which configurations are tested in different scenarios, optimizing CI/CD performance and resource usage. @@ -36,11 +44,15 @@ This allows for flexible control over which configurations are tested in differe Defines the versions of Valkey engine to test against: ```json -[{ "type": "valkey", "version": "7.2.5", "run": "always" }] +[ + { "type": "valkey", "version": "7.2.5", "run": "always" } + // ... other configurations +] ``` - `type`: The type of engine (e.g., Valkey, Redis). -- `version`: The version of the engine. +- `version`: Specifies the engine version that the workflow should checkout. + For example, "7.2.5" represents a release tag, while "7.0" denotes a branch name. The workflow should use this parameter to checkout the specific release version or branch to build the engine with the appropriate version. - `run`: Specifies if the engine version should be tested on every workflow. #### Build Matrix (build-matrix.json) @@ -61,8 +73,8 @@ Defines the host environments for testing: - `OS`: The operating system of the host. - `RUNNER`: The GitHub runner to use. -- `TARGET`: The target environment. -- `run`: Specifies which language workflows should use this host configuration, always means run on each workflow trigger. +- `TARGET`: The target environment as defined in Rust. To see a list of available targets, run `rustup target list`. +- `run`: Specifies which language workflows should use this host configuration. The value `always` indicates that the configuration should be used for every workflow trigger. #### Supported Languages Version (supported-languages-version.json) @@ -81,31 +93,31 @@ Defines the supported versions of languages: - `language`: The language for which the version is supported. - `versions`: The full versions supported of the language which will test against scheduled. -- `always-run-versions`: The versions which will be tested in every workflow run. +- `always-run-versions`: The versions that will always be tested, regardless of the workflow trigger. #### Triggering Workflows -Push to main or create a pull request to run workflows automatically. -Use workflow_dispatch for manual triggers, accepting inputs of full-matrix which is a boolean value to run all configurations. -Scheduled runs are triggered daily to ensure regular testing of all configurations. +- Push to `main` by merging a PR or create a new pull request to run workflows automatically. +- Use `workflow_dispatch` for manual triggers, accepting a boolean configuration parameter to run all configurations. +- Scheduled runs are triggered daily to ensure regular testing of all configurations. ### Mutual vs. Language-Specific Components #### Mutual -`Matrix files` - `.github/json_matrices` -`Shared dependencies installation` - `.github/workflows/install-shared-dependencies/action.yml` -`Linting Rust` - `.github/workflows/lint-rust/action.yml` +- Matrix files - `.github/json_matrices/` +- Shared dependencies installation - `.github/workflows/install-shared-dependencies/action.yml` +- Rust linters - `.github/workflows/lint-rust/action.yml` #### Language-Specific -`Package manager commands` -`Testing frameworks` -`Build processes` +- Package manager commands +- Testing frameworks +- Build processes ### Customizing Workflows -Modify `[language].yml` files to adjust language-specific steps. +Modify `.yml` files to adjust language-specific steps. Update matrix files to change tested versions or environments. Adjust cron schedules in workflow files for different timing of scheduled runs. diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index 4920799ba5..642114e207 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -7,13 +7,16 @@ "TARGET": "x86_64-unknown-linux-gnu", "PACKAGE_MANAGERS": [ "pypi", - "npm" + "npm", + "maven" ], - "run": [ - "always", + "run": "always", + "languages": [ "python", "node", - "java" + "java", + "go", + "dotnet" ] }, { @@ -28,13 +31,16 @@ "TARGET": "aarch64-unknown-linux-gnu", "PACKAGE_MANAGERS": [ "pypi", - "npm" + "npm", + "maven" ], "CONTAINER": "2_28", - "run": [ + "languages": [ "python", "node", - "java" + "java", + "go", + "dotnet" ] }, { @@ -45,12 +51,15 @@ "TARGET": "x86_64-apple-darwin", "PACKAGE_MANAGERS": [ "pypi", - "npm" + "npm", + "maven" ], - "run": [ + "languages": [ "python", "node", - "java" + "java", + "go", + "dotnet" ] }, { @@ -61,12 +70,15 @@ "TARGET": "aarch64-apple-darwin", "PACKAGE_MANAGERS": [ "pypi", - "npm" + "npm", + "maven" ], - "run": [ + "languages": [ "python", "node", - "java" + "java", + "go", + "dotnet" ] }, { @@ -84,7 +96,7 @@ "PACKAGE_MANAGERS": [ "npm" ], - "run": [ + "languages": [ "node" ] }, @@ -99,8 +111,24 @@ "PACKAGE_MANAGERS": [ "npm" ], - "run": [ + "languages": [ "node" ] + }, + { + "OS": "amazon-linux", + "NAMED_OS": "linux", + "RUNNER": "ubuntu-latest", + "ARCH": "x64", + "TARGET": "x86_64-unknown-linux-gnu", + "IMAGE": "amazonlinux:latest", + "PACKAGE_MANAGERS": [], + "languages": [ + "python", + "node", + "java", + "go", + "dotnet" + ] } ] diff --git a/.github/json_matrices/engine-matrix.json b/.github/json_matrices/engine-matrix.json index 149f6f1080..06a8a27fd9 100644 --- a/.github/json_matrices/engine-matrix.json +++ b/.github/json_matrices/engine-matrix.json @@ -1,12 +1,12 @@ [ { "type": "valkey", - "version": "8.0.0", + "version": "8.0", "run": "always" }, { "type": "valkey", - "version": "7.2.5" + "version": "7.2" }, { "type": "redis", diff --git a/.github/json_matrices/supported-languages-versions.json b/.github/json_matrices/supported-languages-versions.json index bc6fd5f472..cc9c9a5eea 100644 --- a/.github/json_matrices/supported-languages-versions.json +++ b/.github/json_matrices/supported-languages-versions.json @@ -6,12 +6,22 @@ }, { "language": "python", - "versions": ["3.8", "3.9", "3.10", "3.11", "3.12"], - "always-run-versions": ["3.8", "3.12"] + "versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + "always-run-versions": ["3.8", "3.13"] }, { "language": "node", - "full-versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], + "versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], "always-run-versions": ["16.x", "20.x"] + }, + { + "language": "dotnet", + "versions": ["8.0", "6.0"], + "always-run-versions": ["8.0"] + }, + { + "language": "go", + "versions": ["1.22.0", "1.18.10"], + "always-run-versions": ["1.22.0"] } ] diff --git a/.github/workflows/build-node-wrapper/action.yml b/.github/workflows/build-node-wrapper/action.yml index f7590c8b0a..9d2f14d59f 100644 --- a/.github/workflows/build-node-wrapper/action.yml +++ b/.github/workflows/build-node-wrapper/action.yml @@ -76,9 +76,9 @@ runs: shell: bash working-directory: ./node run: | - rm -rf node_modules && npm install --frozen-lockfile + rm -rf node_modules && rm -rf package-lock.json && npm install cd rust-client - npm install --frozen-lockfile + rm -rf node_modules && rm -rf package-lock.json && npm install - name: Build shell: bash diff --git a/.github/workflows/build-python-wrapper/action.yml b/.github/workflows/build-python-wrapper/action.yml index 72863c6a43..25c7e20b7d 100644 --- a/.github/workflows/build-python-wrapper/action.yml +++ b/.github/workflows/build-python-wrapper/action.yml @@ -15,7 +15,7 @@ inputs: required: true engine-version: description: "Engine version to install" - required: true + required: false type: string publish: description: "Enable building the wrapper in release mode" diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml index 61516b1cc9..5bd777b5a1 100644 --- a/.github/workflows/create-test-matrices/action.yml +++ b/.github/workflows/create-test-matrices/action.yml @@ -2,10 +2,23 @@ inputs: language-name: description: "Language name" required: true - dispatch-run-full-matrix: + type: choice + options: + - java + - node + - python + - go + - C# + run-full-matrix: description: "Run the full matrix" - required: false - default: "false" + required: true + type: boolean + containers: + description: "Run in containers" + required: true + default: false + type: boolean + outputs: engine-matrix-output: description: "Engine matrix" @@ -15,7 +28,7 @@ outputs: value: ${{ steps.load-host-matrix.outputs.host-matrix }} version-matrix-output: description: "Version matrix" - value: ${{ steps.create-version-matrix.outputs.version-matrix }} + value: ${{ steps.create-lang-version-matrix.outputs.version-matrix }} runs: using: "composite" @@ -24,31 +37,44 @@ runs: id: load-engine-matrix shell: bash run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then - echo "engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + set -o pipefail + echo 'Select server engines to run tests against' + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.run-full-matrix }}" == "false" ]]; then + echo 'Pick engines marked as `"run": "always"` only - on PR, push or manually triggered job which does not require full matrix' + jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json | awk '{ printf "engine-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT else - echo "engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + echo 'Pick all engines - on cron (schedule) or if manually triggered job requires a full matrix' + jq -c . < .github/json_matrices/engine-matrix.json | awk '{ printf "engine-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT fi - echo "engine-matrix=$(cat $GITHUB_OUTPUT)" + cat $GITHUB_OUTPUT - name: Load host matrix id: load-host-matrix shell: bash run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then - echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["always"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + set -o pipefail + [[ "${{ inputs.containers }}" == "true" ]] && CONDITION=".IMAGE?" || CONDITION=".IMAGE == null" + echo 'Select runners (VMs) to run tests on' + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.run-full-matrix }}" == "false" ]]; then + echo 'Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix' + jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/build-matrix.json | awk '{ printf "host-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT else - echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["${{ inputs.language-name }}"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + echo 'Pick all runners assigned for the chosen client (language) - on cron (schedule) or if manually triggered job requires a full matrix' + jq -c "[.[] | select(.languages? and any(.languages[] == \"${{ inputs.language-name }}\"; .) and $CONDITION)]" < .github/json_matrices/build-matrix.json | awk '{ printf "host-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT fi - echo "host-matrix=$(cat $GITHUB_OUTPUT)" + cat $GITHUB_OUTPUT - - name: Create version matrix - id: create-version-matrix + - name: Create language version matrix + id: create-lang-version-matrix shell: bash run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then - echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + set -o pipefail + echo 'Select language (framework/SDK) versions to run tests on' + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.run-full-matrix }}" == "false" ]]; then + echo 'Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix' + jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"]][0] // []' < .github/json_matrices/supported-languages-versions.json | awk '{ printf "version-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT else - echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | (.versions // .["full-versions"]) | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + echo 'Pick language versions listed in 'versions' - on cron (schedule) or if manually triggered job requires a full matrix' + jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .versions][0]' < .github/json_matrices/supported-languages-versions.json | awk '{ printf "version-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT fi - echo "version-matrix=$(cat $GITHUB_OUTPUT)" + cat $GITHUB_OUTPUT diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index c7de56e9ed..1cd5778a5c 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -10,73 +10,83 @@ on: - csharp/** - glide-core/src/** - glide-core/redis-rs/redis/src/** + - utils/cluster_manager.py - .github/workflows/csharp.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** pull_request: paths: - csharp/** - glide-core/src/** - glide-core/redis-rs/redis/src/** + - utils/cluster_manager.py - .github/workflows/csharp.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** workflow_dispatch: + inputs: + full-matrix: + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: + required: false + type: string + description: "(Optional) Test run name" + + workflow_call: permissions: contents: read concurrency: - group: C#-${{ github.head_ref || github.ref }} + group: C#-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + +env: + CARGO_TERM_COLOR: always + jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: dotnet + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} - run-tests: - needs: load-engine-matrix - timeout-minutes: 25 + test-csharp: + needs: get-matrices + timeout-minutes: 35 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - dotnet: - # - '6.0' - - "8.0" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu, - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + dotnet: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} runs-on: ${{ matrix.host.RUNNER }} steps: - uses: actions/checkout@v4 - - name: Set up dotnet ${{ matrix.dotnet }} uses: actions/setup-dotnet@v4 @@ -91,12 +101,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} - - name: Format - working-directory: ./csharp - run: dotnet format --verify-no-changes --verbosity diagnostic - - name: Test dotnet ${{ matrix.dotnet }} - working-directory: ./csharp + working-directory: csharp run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror - uses: ./.github/workflows/test-benchmark @@ -108,22 +114,95 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-reports-dotnet-${{ matrix.dotnet }}-redis-${{ matrix.redis }}-${{ matrix.host.RUNNER }} + name: test-reports-dotnet-${{ matrix.dotnet }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} path: | csharp/TestReport.html benchmarks/results/* utils/clusters/** - # TODO Add amazonlinux + get-containers: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: dotnet + run-full-matrix: true + containers: true + + test-csharp-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + # Don't use generated matrix for dotnet until net6.0 compatibility issues resolved on amazon linux + # dotnet: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + dotnet: ["8.0"] + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options + steps: + - name: Install git + run: | + yum update + yum install -y git tar findutils libicu + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up dotnet ${{ matrix.dotnet }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.dotnet }} + + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test dotnet ${{ matrix.dotnet }} + working-directory: csharp + run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror - lint-rust: + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: test-reports-dotnet-${{ matrix.dotnet }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} + path: | + csharp/TestReport.html + benchmarks/results/* + utils/clusters/** + + lint: timeout-minutes: 10 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - - uses: ./.github/workflows/lint-rust + - name: lint rust + uses: ./.github/workflows/lint-rust with: - cargo-toml-folder: ./csharp/lib + cargo-toml-folder: csharp/lib github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Format + working-directory: csharp + run: dotnet format --verify-no-changes --verbosity diagnostic diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 40d7c77cd7..1d17640188 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -9,66 +9,77 @@ on: paths: - glide-core/src/** - glide-core/redis-rs/redis/src/** + - utils/cluster_manager.py - go/** - .github/workflows/go.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** pull_request: paths: - glide-core/src/** - glide-core/redis-rs/redis/src/** + - utils/cluster_manager.py - go/** - .github/workflows/go.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** workflow_dispatch: + inputs: + full-matrix: + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: + required: false + type: string + description: "(Optional) Test run name" + + workflow_call: concurrency: - group: go-${{ github.head_ref || github.ref }} + group: go-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + +env: + CARGO_TERM_COLOR: always + jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: go + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} - build-and-test-go-client: - needs: load-engine-matrix + test-go: + needs: get-matrices timeout-minutes: 35 strategy: - # Run all jobs fail-fast: false matrix: - go: - - "1.22.0" - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu, - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + go: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -100,14 +111,9 @@ jobs: working-directory: ./go run: make build - - name: Run linters - working-directory: ./go - run: make lint-ci - - name: Run tests working-directory: ./go - run: | - make test + run: make test - uses: ./.github/workflows/test-benchmark with: @@ -118,92 +124,106 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: reports-go-${{ matrix.go }}-redis-${{ matrix.redis }}-${{ matrix.os }} + name: test-report-go-${{ matrix.go }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} path: | utils/clusters/** benchmarks/results/** - build-amazonlinux-latest: - if: github.repository_owner == 'valkey-io' + lint: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/workflows/lint-rust + with: + cargo-toml-folder: go + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v5 + with: + go-version: "1.22.0" + cache-dependency-path: go/go.sum + + - name: Install protoc + uses: ./.github/workflows/install-rust-and-protoc + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install and run linters + working-directory: go + run: | + make install-dev-tools install-build-tools build lint-ci + + get-containers: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: go + run-full-matrix: true + containers: true + + test-go-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 strategy: - # Run all jobs fail-fast: false matrix: - go: - - 1.22.0 - runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 15 + go: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options steps: - name: Install git run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm yum update - yum install -y git - git --version - + yum install -y git tar + git config --global --add safe.directory "$GITHUB_WORKSPACE" + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + cache-dependency-path: go/go.sum - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: Install Go - run: | - yum -y install wget - yum -y install tar - wget https://go.dev/dl/go${{ matrix.go }}.linux-amd64.tar.gz - tar -C /usr/local -xzf go${{ matrix.go }}.linux-amd64.tar.gz - echo "/usr/local/go/bin" >> $GITHUB_PATH - echo "$HOME/go/bin" >> $GITHUB_PATH - - - name: Install tools for Go ${{ matrix.go }} - working-directory: ./go - run: make install-tools-go${{ matrix.go }} - - - name: Set LD_LIBRARY_PATH - run: echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/" >> $GITHUB_ENV - - - name: Build client - working-directory: ./go - run: make build - - - name: Run linters - working-directory: ./go - run: make lint-ci + engine-version: ${{ matrix.engine.version }} - - name: Run tests - working-directory: ./go + - name: Install & build & test + working-directory: go run: | - make test + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/ + make install-tools-go${{ matrix.go }} build test - - name: Upload cluster manager logs + - name: Upload test reports if: always() continue-on-error: true uses: actions/upload-artifact@v4 with: - name: cluster-manager-logs-${{ matrix.go }}-redis-6-amazonlinux + name: test-reports-go-${{ matrix.go }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | utils/clusters/** - - lint-rust: - timeout-minutes: 15 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: ./.github/workflows/lint-rust - with: - cargo-toml-folder: ./go - name: lint go rust + benchmarks/results/** diff --git a/.github/workflows/install-valkey/action.yml b/.github/workflows/install-engine/action.yml similarity index 60% rename from .github/workflows/install-valkey/action.yml rename to .github/workflows/install-engine/action.yml index 36da82f93b..48b269634f 100644 --- a/.github/workflows/install-valkey/action.yml +++ b/.github/workflows/install-engine/action.yml @@ -1,4 +1,4 @@ -name: Install Valkey +name: Install Engine inputs: engine-version: @@ -19,16 +19,15 @@ inputs: env: CARGO_TERM_COLOR: always - VALKEY_MIN_VERSION: "7.2.5" runs: using: "composite" + # TODO: self-hosted runners are actually cloning the repo, using the cache from the previous run + # will not work as expected. We need to find a way to cache the valkey repo on the runner itself. steps: - - name: - Cache Valkey - # TODO: remove the musl ARM64 limitation when https://github.com/actions/runner/issues/801 is resolved - if: ${{ inputs.target != 'aarch64-unknown-linux-musl' }} + - name: Cache Valkey + if: ${{!contains(inputs.target, 'aarch64') }} uses: actions/cache@v4 id: cache-valkey with: @@ -36,22 +35,44 @@ runs: ~/valkey key: valkey-${{ inputs.engine-version }}-${{ inputs.target }} + - name: Build Valkey for ARM + if: ${{ contains(inputs.target, 'aarch64') }} + shell: bash + working-directory: ~ + run: | + cd ~ + echo "Building valkey ${{ inputs.engine-version }}" + # check if the valkey repo is already cloned + if [[ ! -d valkey ]]; then + git clone https://github.com/valkey-io/valkey.git + else + # check if the branch=version is already checked out + if [[ $(git branch --show-current) != ${{ inputs.engine-version }} ]]; then + cd valkey + make clean + make distclean + sudo rm -rf /usr/local/bin/redis-* /usr/local/bin/valkey-* ./valkey-* ./redis-* ./dump.rdb + git fetch --all + git checkout ${{ inputs.engine-version }} + git pull + fi + fi + # if no cache hit, build the engine - name: Build Valkey - if: ${{ steps.cache-valkey.outputs.cache-hit != 'true' }} + if: ${{ steps.cache-valkey.outputs.cache-hit != 'true' && !contains(inputs.target, 'aarch64') }} shell: bash run: | echo "Building valkey ${{ inputs.engine-version }}" cd ~ - rm -rf valkey git clone https://github.com/valkey-io/valkey.git cd valkey git checkout ${{ inputs.engine-version }} - make BUILD_TLS=yes - - name: Install Valkey + - name: Install engine shell: bash run: | cd ~/valkey + make BUILD_TLS=yes if command -v sudo &> /dev/null then echo "sudo command exists" @@ -62,6 +83,8 @@ runs: fi echo 'export PATH=/usr/local/bin:$PATH' >>~/.bash_profile + # TODO: This seems redundant to me. Is it necessary? Do we check that the Python we install is the correct version? + # Why here and not elsewhere? All Git git repos were created equal - name: Verify Valkey installation and symlinks if: ${{ !contains(inputs.engine-version, '-rc') }} shell: bash diff --git a/.github/workflows/install-redis/action.yml b/.github/workflows/install-redis/action.yml deleted file mode 100644 index 8c8d707aa0..0000000000 --- a/.github/workflows/install-redis/action.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Install Redis - -inputs: - redis-version: - description: "redis version to install" - required: true - type: string - -env: - CARGO_TERM_COLOR: always - -runs: - using: "composite" - - steps: - - run: mkdir -p ~/redis-binaries/${{ inputs.redis-version }} - shell: bash - - - uses: actions/checkout@v4 - - - - uses: actions/cache@v3 - id: cache-redis - with: - path: | - ~/redis-binaries/${{ inputs.redis-version }}/redis-cli - ~/redis-binaries/${{ inputs.redis-version }}/redis-server - key: ${{ runner.os }}-${{ inputs.redis-version }}-install-redis - - - name: Install redis - shell: bash - if: steps.cache-redis.outputs.cache-hit != 'true' - run: | - sudo apt-get update - wget https://github.com/redis/redis/archive/${{ inputs.redis-version }}.tar.gz; - tar -xzvf ${{ inputs.redis-version }}.tar.gz; - pushd redis-${{ inputs.redis-version }} && BUILD_TLS=yes make && sudo mv src/redis-server src/redis-cli ~/redis-binaries/${{ inputs.redis-version }} && popd; - - - name: Remove the source package - shell: bash - if: steps.cache-redis.outputs.cache-hit != 'true' - run: sudo rm -r redis-${{ inputs.redis-version }} - - - name: Copy executable to place - shell: bash - run: sudo cp ~/redis-binaries/${{ inputs.redis-version }}/redis-server ~/redis-binaries/${{ inputs.redis-version }}/redis-cli /usr/bin/ diff --git a/.github/workflows/install-shared-dependencies/action.yml b/.github/workflows/install-shared-dependencies/action.yml index 49bc44cb24..1e64d939a3 100644 --- a/.github/workflows/install-shared-dependencies/action.yml +++ b/.github/workflows/install-shared-dependencies/action.yml @@ -69,9 +69,9 @@ runs: target: ${{ inputs.target }} github-token: ${{ inputs.github-token }} - - name: Install Valkey - if: ${{ inputs.engine-version != '' }} - uses: ./.github/workflows/install-valkey + - name: Install engine + if: "${{ inputs.engine-version }}" + uses: ./.github/workflows/install-engine with: engine-version: ${{ inputs.engine-version }} target: ${{ inputs.target }} diff --git a/.github/workflows/java-cd.yml b/.github/workflows/java-cd.yml index 3263dd3a6f..215ac6248e 100644 --- a/.github/workflows/java-cd.yml +++ b/.github/workflows/java-cd.yml @@ -241,7 +241,7 @@ jobs: uses: ./.github/workflows/install-shared-dependencies with: os: ${{ matrix.host.OS }} - engine-version: "7.2.5" + engine-version: "7.2" target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index af792539f0..efed8d9da4 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -10,42 +10,53 @@ on: - glide-core/src/** - glide-core/redis-rs/redis/src/** - java/** + - utils/cluster_manager.py - .github/workflows/java.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + pull_request: paths: - glide-core/src/** - glide-core/redis-rs/redis/src/** - java/** + - utils/cluster_manager.py - .github/workflows/java.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: required: false - default: "false" + type: string + description: "(Optional) Test run name" - schedule: - - cron: "0 0 * * *" + workflow_call: concurrency: - group: java-${{ github.head_ref || github.ref }} + group: java-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -56,18 +67,18 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: java - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} test-java: needs: get-matrices timeout-minutes: 35 strategy: - # Run all jobs fail-fast: false matrix: java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -121,40 +132,52 @@ jobs: benchmarks/results/** java/client/build/reports/spotbugs/** - build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') - needs: get-matrices + get-containers: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: java + run-full-matrix: true + containers: true + + test-java-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 strategy: - # Run all jobs fail-fast: false matrix: - java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} - runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 35 + java: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options steps: - name: Install git run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm yum update - yum install -y git - git --version - + yum install -y git tar java-${{ matrix.java }}-amazon-corretto-devel.x86_64 + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` - uses: actions/checkout@v4 - - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive + with: + submodules: recursive - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} @@ -164,10 +187,6 @@ jobs: version: "26.1" repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install Java - run: | - yum install -y java-${{ matrix.java }}-amazon-corretto-devel.x86_64 - - name: Build java wrapper working-directory: java run: ./gradlew --continue build -x javadoc @@ -177,7 +196,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-reports-${{ matrix.java }}-${{ matrix.engine }}-amazon-linux + name: test-reports-java-${{ matrix.java }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | java/client/build/reports/** java/integTest/build/reports/** @@ -192,7 +211,8 @@ jobs: - uses: ./.github/workflows/lint-rust with: - cargo-toml-folder: ./java + cargo-toml-folder: java + github-token: ${{ secrets.GITHUB_TOKEN }} name: lint java rust test-modules: diff --git a/.github/workflows/lint-rust/action.yml b/.github/workflows/lint-rust/action.yml index 198c04de1c..863ad305bc 100644 --- a/.github/workflows/lint-rust/action.yml +++ b/.github/workflows/lint-rust/action.yml @@ -12,7 +12,6 @@ inputs: runs: using: "composite" - steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..06f5065e24 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,88 @@ +name: Nightly tests + +on: + workflow_dispatch: # note: if started manually, it won't run all matrix + inputs: + full-matrix: + description: "Run the full engine and host matrix" + type: boolean + default: false + # GHA supports up to 10 inputs, there is no option for multi-choice + core: + description: "Test GLIDE core" + type: boolean + default: true + node: + description: "Test Node client" + type: boolean + default: true + python: + description: "Test Python client" + type: boolean + default: true + java: + description: "Test Java client" + type: boolean + default: true + csharp: + description: "Test C# client" + type: boolean + default: false + go: + description: "Test Golang client" + type: boolean + default: false + + schedule: # running tests by cron is disabled on forks by the condition defined for each job + - cron: "0 3 * * *" # Runs at 03:00 (3 AM) UTC every day + +concurrency: + group: nightly-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} + cancel-in-progress: true + +# TODO matrix by workflow (`uses`) - not supported yet by GH +jobs: + check-input: + runs-on: ubuntu-latest + steps: + - name: no tests selected + run: false + if: github.event_name == 'workflow_dispatch' && inputs.core == false && inputs.java == false && inputs.python == false && inputs.node == false && inputs.java == false && inputs.csharp == false && inputs.go == false + + run-full-tests-for-core: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.core == true) + uses: ./.github/workflows/rust.yml + name: Run CI for GLIDE core lib + secrets: inherit + + run-full-tests-for-java: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.java == true) + uses: ./.github/workflows/java.yml + name: Run CI for java client + secrets: inherit + + run-full-tests-for-python: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.python == true) + uses: ./.github/workflows/python.yml + name: Run CI for python client + secrets: inherit + + run-full-tests-for-node: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.node == true) + uses: ./.github/workflows/node.yml + name: Run CI for node client + secrets: inherit + + run-full-tests-for-csharp: + # C# deactivated in cron, uncomment condition to activate + #if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.csharp == true) + if: (github.event_name == 'workflow_dispatch' && inputs.csharp == true) + uses: ./.github/workflows/csharp.yml + name: Run CI for c# client + secrets: inherit + + run-full-tests-for-go: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.go == true) + uses: ./.github/workflows/go.yml + name: Run CI for go client + secrets: inherit diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 8f3988120a..4c0b9a2eec 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -16,9 +16,9 @@ on: - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/install-engine/action.yml + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml pull_request: paths: - glide-core/src/** @@ -30,31 +30,36 @@ on: - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/install-engine/action.yml + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: required: false - default: "false" + type: string + description: "(Optional) Test run name" - schedule: - - cron: "0 1 * * *" + workflow_call: concurrency: - group: node-${{ github.head_ref || github.ref }} + group: node-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true env: CARGO_TERM_COLOR: always +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -66,10 +71,10 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: node - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} test-node: - runs-on: ubuntu-latest + runs-on: ${{ matrix.host.RUNNER }} needs: [get-matrices] timeout-minutes: 25 strategy: @@ -77,7 +82,7 @@ jobs: matrix: engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} - node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output)}} + node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} steps: - uses: actions/checkout@v4 @@ -92,6 +97,8 @@ jobs: uses: ./.github/workflows/build-node-wrapper with: os: ${{ matrix.host.OS }} + named_os: ${{ matrix.host.NAMED_OS }} + arch: ${{ matrix.host.ARCH }} target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} @@ -102,18 +109,16 @@ jobs: - name: test hybrid node modules - commonjs run: | - npm install --package-lock-only - npm ci - npm run build-and-test + npm install + npm run test working-directory: ./node/hybrid-node-tests/commonjs-test env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-commonjs.html - name: test hybrid node modules - ecma run: | - npm install --package-lock-only - npm ci - npm run build-and-test + npm install + npm run test working-directory: ./node/hybrid-node-tests/ecmascript-test env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-ecma.html @@ -127,7 +132,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.node }}-ubuntu + name: test-report-node-${{ matrix.node }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.OS }}-${{ matrix.host.ARCH }} path: | node/test-report*.html utils/clusters/** @@ -145,86 +150,74 @@ jobs: cargo-toml-folder: ./node/rust-client github-token: ${{ secrets.GITHUB_TOKEN }} - build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') + get-containers: runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 15 - steps: - - name: Install git - run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm - yum install -y git - git --version + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + steps: - uses: actions/checkout@v4 - - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive - - - name: Install NodeJS - run: | - yum install -y nodejs - - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: test - run: npm test - working-directory: ./node - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices with: - name: test-report-node-amazonlinux - path: | - node/test-report*.html - utils/clusters/** - benchmarks/results/** + language-name: node + run-full-matrix: true + containers: true - build-and-test-linux-musl-on-x86: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') - name: Build and test Node wrapper on Linux musl - runs-on: ubuntu-latest + test-node-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + node: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} container: - image: node:alpine - options: --user root --privileged - + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options steps: - name: Install git run: | - apk update - apk add git - + if [[ ${{ contains(matrix.host.TARGET, 'musl') }} == true ]]; then + apk update + apk add --no-cache git tar + elif [[ ${{ contains(matrix.host.IMAGE, 'amazonlinux') }} == true ]]; then + yum update + yum install -y git tar + fi + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` - uses: actions/checkout@v4 - name: Setup musl on Linux + if: ${{ contains(matrix.host.TARGET, 'musl') }} uses: ./.github/workflows/setup-musl-on-linux with: workspace: $GITHUB_WORKSPACE npm-scope: ${{ secrets.NPM_SCOPE }} npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} + - name: Setup Node + uses: actions/setup-node@v4 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + with: + node-version: ${{ matrix.node }} + - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: - os: ubuntu - named_os: linux - arch: x64 - target: x86_64-unknown-linux-musl + os: ${{ matrix.host.OS }} + named_os: ${{ matrix.host.NAMED_OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" + engine-version: ${{ matrix.engine.version }} + arch: ${{ matrix.host.ARCH }} - name: test run: npm test @@ -235,7 +228,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-report-node-linux-musl + name: test-report-node-${{ matrix.node }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | node/test-report*.html utils/clusters/** diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index e2c7b42ff3..9667fe8cc0 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -10,8 +10,9 @@ on: - .github/workflows/start-self-hosted-runner/action.yml - .github/workflows/install-rust-and-protoc/action.yml - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml push: tags: - "v*.*" @@ -22,7 +23,7 @@ on: required: true concurrency: - group: npm-${{ github.head_ref || github.ref }} + group: node-cd-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true permissions: @@ -59,7 +60,7 @@ jobs: shell: bash run: | # Get the matrix from the matrix.json file, without the object that has the IMAGE key - export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["npm"])))' < .github/json_matrices/build-matrix.json | jq -c .)" + export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["npm"]))))' < .github/json_matrices/build-matrix.json | jq -c .)" echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT publish-binaries: @@ -153,7 +154,6 @@ jobs: npm_scope: ${{ vars.NPM_SCOPE }} publish: "true" github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - name: Check if RC and set a distribution tag for the package shell: bash @@ -260,7 +260,6 @@ jobs: os: ubuntu target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - name: Check if RC and set a distribution tag for the package shell: bash diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index 5d547ac7f5..647dbd1c8e 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -9,8 +9,9 @@ on: - .github/workflows/build-python-wrapper/action.yml - .github/workflows/start-self-hosted-runner/action.yml - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/install-engine/action.yml + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml push: tags: - "v*.*" @@ -41,7 +42,7 @@ jobs: shell: bash run: | # Get the matrix from the matrix.json file, without the object that has the IMAGE key - export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["pypi"])))' < .github/json_matrices/build-matrix.json | jq -c .)" + export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["pypi"]))))' < .github/json_matrices/build-matrix.json | jq -c .)" echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT start-self-hosted-runner: @@ -114,13 +115,6 @@ jobs: with: python-version: "3.12" - - name: Setup Python for self-hosted Ubuntu runners - if: contains(matrix.build.RUNNER, 'self-hosted') - run: | - sudo apt update -y - sudo apt upgrade -y - sudo apt install python3 python3-venv python3-pip -y - - name: Update package version in config.toml uses: ./.github/workflows/update-glide-version with: @@ -134,7 +128,6 @@ jobs: target: ${{ matrix.build.TARGET }} publish: "true" github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2" - name: Include protobuf files in the package working-directory: ./python @@ -150,7 +143,7 @@ jobs: with: working-directory: ./python target: ${{ matrix.build.TARGET }} - args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12' || 'python3.10' }} + args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }} manylinux: auto container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }} before-script-linux: | @@ -178,9 +171,10 @@ jobs: if: startsWith(matrix.build.NAMED_OS, 'darwin') uses: PyO3/maturin-action@v1 with: + maturin-version: latest working-directory: ./python target: ${{ matrix.build.TARGET }} - args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12' || 'python3.12' }} + args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }} - name: Upload Python wheels if: github.event_name != 'pull_request' @@ -220,10 +214,6 @@ jobs: matrix: build: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }} steps: - - name: Setup self-hosted runner access - if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }} - run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - - name: checkout uses: actions/checkout@v4 @@ -232,8 +222,8 @@ jobs: with: python-version: 3.12 - - name: Install ValKey - uses: ./.github/workflows/install-valkey + - name: Install engine + uses: ./.github/workflows/install-engine with: version: "8.0" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index abf563b98a..2bdca043d1 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -16,10 +16,10 @@ on: - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/json_matrices/engine-matrix.json + - .github/workflows/install-engine/action.yml - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** pull_request: paths: @@ -32,40 +32,52 @@ on: - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/json_matrices/engine-matrix.json + - .github/workflows/install-engine/action.yml - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: required: false - default: "false" + type: string + description: "(Optional) Test run name" - schedule: - - cron: "0 2 * * *" + workflow_call: concurrency: - group: python-${{ github.head_ref || github.ref }} + group: python-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true +permissions: + contents: read + # Allows the GITHUB_TOKEN to make an API call to generate an OIDC token. + id-token: write + +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} - version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output}} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - uses: actions/checkout@v4 - id: get-matrices uses: ./.github/workflows/create-test-matrices with: language-name: python - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} test-python: runs-on: ${{ matrix.host.RUNNER }} @@ -74,9 +86,9 @@ jobs: strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} steps: - uses: actions/checkout@v4 @@ -131,6 +143,8 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** + + # run pubsub tests in another job - they take too much time test-pubsub-python: runs-on: ${{ matrix.host.RUNNER }} needs: get-matrices @@ -138,8 +152,8 @@ jobs: strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 @@ -182,7 +196,8 @@ jobs: - name: lint rust uses: ./.github/workflows/lint-rust with: - cargo-toml-folder: ./python + cargo-toml-folder: python + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies if: always() @@ -210,53 +225,74 @@ jobs: working-directory: ./python run: | black --check --diff . - build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') + + get-containers: runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 15 - steps: - - name: Install git - run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm - yum install -y git - git --version + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + steps: - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: python + run-full-matrix: true + containers: true - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive - - - name: Install python + test-python-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + # Don't use generated matrix for python until compatibility issues resolved on amazon linux + # python: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options + steps: + - name: Install git and python run: | - yum install -y python3 + yum update + yum install -y git tar python3 + python3 -m ensurepip --upgrade + python3 -m pip install --upgrade pip + python3 -m pip install mypy-protobuf virtualenv + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` + - uses: actions/checkout@v4 - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" + os: ubuntu + target: aarch64-unknown-linux-gnu github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - name: Test compatibility with pytest + - name: Test with pytest working-directory: ./python run: | source .env/bin/activate - pytest --asyncio-mode=auto -m smoke_test --html=pytest_report.html --self-contained-html + cd python/tests/ + pytest --asyncio-mode=auto --html=pytest_report.html --self-contained-html - name: Upload test reports if: always() continue-on-error: true uses: actions/upload-artifact@v4 with: - name: smoke-test-report-amazon-linux + name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | python/python/tests/pytest_report.html + utils/clusters/** + benchmarks/results/** test-modules: if: (github.repository_owner == 'valkey-io' && github.event_name == 'workflow_dispatch') || github.event.pull_request.head.repo.owner.login == 'valkey-io' @@ -271,6 +307,8 @@ jobs: run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - uses: actions/checkout@v4 + with: + submodules: recursive - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper @@ -278,7 +316,6 @@ jobs: os: ubuntu target: aarch64-unknown-linux-gnu github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Test with pytest working-directory: ./python run: | diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 02274c06e0..0c71fa2f86 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,9 +13,10 @@ on: - utils/cluster_manager.py - .github/workflows/rust.yml - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/install-valkey/action.yml + - .github/workflows/install-engine/action.yml - .github/workflows/lint-rust/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** - deny.toml pull_request: paths: @@ -25,41 +26,63 @@ on: - utils/cluster_manager.py - .github/workflows/rust.yml - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/install-valkey/action.yml + - .github/workflows/install-engine/action.yml - .github/workflows/lint-rust/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** - deny.toml workflow_dispatch: + inputs: + full-matrix: + description: "Run the full engine and host matrix" + type: boolean + default: false + name: + required: false + type: string + description: "(Optional) Test run name" + + workflow_call: concurrency: - group: rust-${{ github.head_ref || github.ref }} + group: rust-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true env: CARGO_TERM_COLOR: always +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') || github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + # language version matrix is omitted - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: rust + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} - build: - runs-on: ubuntu-latest - needs: load-engine-matrix + tests: + runs-on: ${{ matrix.host.RUNNER }} + needs: get-matrices timeout-minutes: 15 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} steps: - uses: actions/checkout@v4 diff --git a/glide-core/src/client/types.rs b/glide-core/src/client/types.rs index 2fa46037da..ef4be661e6 100644 --- a/glide-core/src/client/types.rs +++ b/glide-core/src/client/types.rs @@ -2,7 +2,9 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ +#[allow(unused_imports)] use logger_core::log_warn; +#[allow(unused_imports)] use std::collections::HashSet; use std::time::Duration; diff --git a/glide-core/tests/test_standalone_client.rs b/glide-core/tests/test_standalone_client.rs index 5b269dd42c..8001ccab0c 100644 --- a/glide-core/tests/test_standalone_client.rs +++ b/glide-core/tests/test_standalone_client.rs @@ -251,7 +251,7 @@ mod standalone_client_tests { .map(|mock| mock.get_number_of_received_commands()) .collect(); replica_reads.sort(); - assert_eq!(config.expected_replica_reads, replica_reads); + assert!(config.expected_replica_reads <= replica_reads); } #[rstest] diff --git a/glide-core/tests/utilities/mod.rs b/glide-core/tests/utilities/mod.rs index 4e73c0ea93..765c1cffb1 100644 --- a/glide-core/tests/utilities/mod.rs +++ b/glide-core/tests/utilities/mod.rs @@ -734,7 +734,7 @@ pub async fn kill_connection(client: &mut impl glide_core::client::GlideClientFo &client_kill_cmd, Some(RoutingInfo::MultiNode(( MultipleNodeRoutingInfo::AllNodes, - None, + Some(redis::cluster_routing::ResponsePolicy::AllSucceeded), ))), ) .await diff --git a/node/hybrid-node-tests/commonjs-test/package.json b/node/hybrid-node-tests/commonjs-test/package.json index f45541c135..6e1291a680 100644 --- a/node/hybrid-node-tests/commonjs-test/package.json +++ b/node/hybrid-node-tests/commonjs-test/package.json @@ -19,8 +19,8 @@ "valkey-glide": "file:../../../node/build-ts/cjs" }, "devDependencies": { - "@types/node": "^18.7.9", - "prettier": "^2.8.8", - "typescript": "^4.8.4" + "@types/node": "^22.8", + "prettier": "^3.3", + "typescript": "^5.6" } } diff --git a/node/hybrid-node-tests/ecmascript-test/package.json b/node/hybrid-node-tests/ecmascript-test/package.json index df578128a7..d1d91d5a97 100644 --- a/node/hybrid-node-tests/ecmascript-test/package.json +++ b/node/hybrid-node-tests/ecmascript-test/package.json @@ -19,8 +19,8 @@ "valkey-glide": "file:../../../node/build-ts/mjs" }, "devDependencies": { - "@types/node": "^18.7.9", - "prettier": "^2.8.8", - "typescript": "^4.8.4" + "@types/node": "^22.8", + "prettier": "^3.3", + "typescript": "^5.6" } } diff --git a/node/jest.config.js b/node/jest.config.js index e73e7ce439..6952aecfca 100644 --- a/node/jest.config.js +++ b/node/jest.config.js @@ -1,7 +1,9 @@ /* eslint no-undef: off */ module.exports = { preset: "ts-jest", - transform: { "^.+\\.ts?$": "ts-jest" }, + transform: { + "^.+\\.(t|j)s$": ["ts-jest", { isolatedModules: true }], + }, testEnvironment: "node", testRegex: "/tests/.*\\.(test|spec)?\\.(ts|tsx)$", moduleFileExtensions: [ @@ -27,4 +29,5 @@ module.exports = { }, ], ], + setupFilesAfterEnv: ["./tests/setup.js"], }; diff --git a/node/package.json b/node/package.json index 06a2dc7743..685d1338fd 100644 --- a/node/package.json +++ b/node/package.json @@ -32,15 +32,15 @@ "compile-protobuf-files": "cd src && pbjs -t static-module -o ProtobufMessage.js ../../glide-core/src/protobuf/*.proto && pbts -o ProtobufMessage.d.ts ProtobufMessage.js", "clean": "rm -rf build-ts rust-client/target docs glide-logs rust-client/glide-rs.*.node rust-client/index.* src/ProtobufMessage.*", "fix-protobuf-file": "replace 'this\\.encode\\(message, writer\\)\\.ldelim' 'this.encode(message, writer && writer.len ? writer.fork() : writer).ldelim' src/ProtobufMessage.js", - "test": "npm run build-test-utils && jest --verbose --runInBand --testPathIgnorePatterns='ServerModules'", + "test": "npm run build-test-utils && jest --verbose --testPathIgnorePatterns='ServerModules'", "test-minimum": "npm run build-test-utils && jest --verbose --runInBand --testNamePattern='^(.(?!(GlideJson|GlideFt|pubsub|kill)))*$'", - "test-modules": "npm run build-test-utils && jest --verbose --runInBand --testNamePattern='(GlideJson|GlideFt)'", + "test-modules": "npm run build-test-utils && jest --verbose --testNamePattern='(GlideJson|GlideFt)'", "build-test-utils": "cd ../utils && npm i && npm run build", "lint:fix": "npm run install-linting && npx eslint -c ../eslint.config.mjs --fix && npm run prettier:format", "lint": "npm run install-linting && npx eslint -c ../eslint.config.mjs && npm run prettier:check:ci", "install-linting": "cd ../ & npm install", "prepack": "npmignore --auto", - "prereq": "git submodule update --init --recursive && npm install", + "prereq": "npm install", "prettier:check:ci": "npx prettier --check . --ignore-unknown '!**/*.{js,d.ts}'", "prettier:format": "npx prettier --write . --ignore-unknown '!**/*.{js,d.ts}'" }, diff --git a/node/tests/AsyncClient.test.ts b/node/tests/AsyncClient.test.ts index 6d0d3b4246..c810b07f91 100644 --- a/node/tests/AsyncClient.test.ts +++ b/node/tests/AsyncClient.test.ts @@ -35,8 +35,8 @@ describe("AsyncClient", () => { await flushallOnPort(port); }); - afterAll(() => { - server.close(); + afterAll(async () => { + await server.close(); }); runCommonTests({ diff --git a/node/tests/GlideClient.test.ts b/node/tests/GlideClient.test.ts index 7e0a75ddec..e1043c0657 100644 --- a/node/tests/GlideClient.test.ts +++ b/node/tests/GlideClient.test.ts @@ -20,6 +20,7 @@ import { GlideClient, GlideRecord, GlideString, + ListDirection, ProtocolVersion, RequestError, Script, @@ -34,12 +35,12 @@ import { convertStringArrayToBuffer, createLongRunningLuaScript, createLuaLibWithLongRunningFunction, - DumpAndRestureTest, + DumpAndRestoreTest, encodableTransactionTest, - encodedTransactionTest, flushAndCloseClient, generateLuaLibCode, getClientConfigurationOption, + getServerVersion, parseCommandLineArgs, parseEndpoints, transactionTest, @@ -58,10 +59,12 @@ describe("GlideClient", () => { const standaloneAddresses = parseCommandLineArgs()["standalone-endpoints"]; cluster = standaloneAddresses - ? await RedisCluster.initFromExistingCluster( + ? await ValkeyCluster.initFromExistingCluster( + false, parseEndpoints(standaloneAddresses), + getServerVersion, ) - : await RedisCluster.createCluster(false, 1, 1); + : await ValkeyCluster.createCluster(false, 1, 1, getServerVersion); }, 20000); afterEach(async () => { @@ -71,6 +74,8 @@ describe("GlideClient", () => { afterAll(async () => { if (testsFailed === 0) { await cluster.close(); + } else { + await cluster.close(true); } }, TIMEOUT); @@ -136,11 +141,9 @@ describe("GlideClient", () => { "check that blocking commands returns never timeout_%p", async (protocol) => { client = await GlideClient.createClient( - getClientConfigurationOption( - cluster.getAddresses(), - protocol, - 300, - ), + getClientConfigurationOption(cluster.getAddresses(), protocol, { + requestTimeout: 300, + }), ); const promiseList = [ @@ -277,25 +280,6 @@ describe("GlideClient", () => { }, ); - it.each([ProtocolVersion.RESP2, ProtocolVersion.RESP3])( - `can get Bytes decoded transactions_%p`, - async (protocol) => { - client = await GlideClient.createClient( - getClientConfigurationOption(cluster.getAddresses(), protocol), - ); - const transaction = new Transaction(); - const expectedRes = await encodedTransactionTest(transaction); - transaction.select(0); - const result = await client.exec(transaction, { - decoder: Decoder.Bytes, - }); - expectedRes.push(["select(0)", "OK"]); - - validateTransactionResponse(result, expectedRes); - client.close(); - }, - ); - it.each([ProtocolVersion.RESP2, ProtocolVersion.RESP3])( `dump and restore transactions_%p`, async (protocol) => { @@ -303,9 +287,12 @@ describe("GlideClient", () => { getClientConfigurationOption(cluster.getAddresses(), protocol), ); const bytesTransaction = new Transaction(); - const expectedBytesRes = await DumpAndRestureTest( + await client.set("key", "value"); + const dumpValue: Buffer = (await client.dump("key")) as Buffer; + await client.del(["key"]); + const expectedBytesRes = await DumpAndRestoreTest( bytesTransaction, - Buffer.from("value"), + dumpValue, ); bytesTransaction.select(0); const result = await client.exec(bytesTransaction, { @@ -316,14 +303,14 @@ describe("GlideClient", () => { validateTransactionResponse(result, expectedBytesRes); const stringTransaction = new Transaction(); - await DumpAndRestureTest(stringTransaction, "value"); + await DumpAndRestoreTest(stringTransaction, dumpValue); stringTransaction.select(0); // Since DUMP gets binary results, we cannot use the string decoder here, so we expected to get an error. await expect( client.exec(stringTransaction, { decoder: Decoder.String }), ).rejects.toThrowError( - "invalid utf-8 sequence of 1 bytes from index 9", + /invalid utf-8 sequence of 1 bytes from index/, ); client.close(); diff --git a/node/tests/GlideClientInternals.test.ts b/node/tests/GlideClientInternals.test.ts index 9d8f093d0a..b12fd007d4 100644 --- a/node/tests/GlideClientInternals.test.ts +++ b/node/tests/GlideClientInternals.test.ts @@ -2,7 +2,7 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { beforeAll, describe, expect, it } from "@jest/globals"; +import { describe, expect, it } from "@jest/globals"; import fs from "fs"; import { createLeakedArray, @@ -32,7 +32,6 @@ import { GlideReturnType, InfoOptions, isGlideRecord, - Logger, RequestError, SlotKeyTypes, TimeUnit, @@ -45,10 +44,6 @@ import { import { convertStringArrayToBuffer } from "./TestUtilities"; const { RequestType, CommandRequest } = command_request; -beforeAll(() => { - Logger.init("info"); -}); - enum ResponseType { /** Type of a response that returns a null. */ Null, diff --git a/node/tests/GlideClusterClient.test.ts b/node/tests/GlideClusterClient.test.ts index eea642aef5..3707b46f74 100644 --- a/node/tests/GlideClusterClient.test.ts +++ b/node/tests/GlideClusterClient.test.ts @@ -47,6 +47,7 @@ import { generateLuaLibCode, getClientConfigurationOption, getFirstResult, + getServerVersion, intoArray, intoString, parseCommandLineArgs, @@ -67,17 +68,13 @@ describe("GlideClusterClient", () => { const clusterAddresses = parseCommandLineArgs()["cluster-endpoints"]; // Connect to cluster or create a new one based on the parsed addresses cluster = clusterAddresses - ? (await RedisCluster.initFromExistingCluster( + ? await ValkeyCluster.initFromExistingCluster( + true, parseEndpoints(clusterAddresses), - )) - ? await ValkeyCluster.initFromExistingCluster( - true, - parseEndpoints(clusterAddresses), - getServerVersion, - ) - : // setting replicaCount to 1 to facilitate tests routed to replicas - await RedisCluster.createCluster(true, 3, 1) - : await ValkeyCluster.createCluster(true, 3, 1, getServerVersion); + getServerVersion, + ) + : // setting replicaCount to 1 to facilitate tests routed to replicas + await ValkeyCluster.createCluster(true, 3, 1, getServerVersion); }, 20000); afterEach(async () => { @@ -87,6 +84,8 @@ describe("GlideClusterClient", () => { afterAll(async () => { if (testsFailed === 0) { await cluster.close(); + } else { + await cluster.close(true); } }); @@ -249,7 +248,7 @@ describe("GlideClusterClient", () => { expect(await client.set(key, value)).toEqual("OK"); // Since DUMP gets binary results, we cannot use the default decoder (string) here, so we expected to get an error. await expect(client.customCommand(["DUMP", key])).rejects.toThrow( - "invalid utf-8 sequence of 1 bytes from index 9", + "invalid utf-8 sequence of 1 bytes from index", ); const dumpResult = await client.customCommand(["DUMP", key], { @@ -377,10 +376,14 @@ describe("GlideClusterClient", () => { if (!cluster.checkIfServerVersionLessThan("7.0.0")) { lmpopArr.push( - client.lmpop(["abc", "def"], ListDirection.LEFT, 1), + client.lmpop(["abc", "def"], ListDirection.LEFT, { + count: 1, + }), ); lmpopArr.push( - client.blmpop(["abc", "def"], ListDirection.RIGHT, 0.1, 1), + client.blmpop(["abc", "def"], ListDirection.RIGHT, 0.1, { + count: 1, + }), ); } @@ -450,9 +453,15 @@ describe("GlideClusterClient", () => { ); } - for (const promise of promises) { - await expect(promise).rejects.toThrowError(/crossslot/i); - } + await Promise.allSettled(promises).then((results) => { + results.forEach((result) => { + expect(result.status).toBe("rejected"); + + if (result.status === "rejected") { + expect(result.reason.message).toContain("CrossSlot"); + } + }); + }); client.close(); }, diff --git a/node/tests/PubSub.test.ts b/node/tests/PubSub.test.ts index 5e8d40207a..8e87b7ee1c 100644 --- a/node/tests/PubSub.test.ts +++ b/node/tests/PubSub.test.ts @@ -80,12 +80,22 @@ describe("PubSub", () => { : await ValkeyCluster.createCluster(true, 3, 1, getServerVersion); }, 40000); afterEach(async () => { - await flushAndCloseClient(false, cmdCluster.getAddresses()); - await flushAndCloseClient(true, cmeCluster.getAddresses()); + if (cmdCluster) { + await flushAndCloseClient(false, cmdCluster.getAddresses()); + } + + if (cmeCluster) { + await flushAndCloseClient(true, cmeCluster.getAddresses()); + } }); afterAll(async () => { - await cmeCluster.close(); - await cmdCluster.close(); + if (cmdCluster) { + await cmdCluster.close(); + } + + if (cmeCluster) { + await cmeCluster.close(); + } }); async function createClients( @@ -3956,6 +3966,12 @@ describe("PubSub", () => { let client2: TGlideClient | null = null; try { + const minVersion = "7.0.0"; + + if (cmeCluster.checkIfServerVersionLessThan(minVersion)) { + return; // Skip test if server version is less than required + } + const regularChannel = "regular_channel"; const shardChannel = "shard_channel"; @@ -3977,12 +3993,6 @@ describe("PubSub", () => { pubSub, ); - const minVersion = "7.0.0"; - - if (cmeCluster.checkIfServerVersionLessThan(minVersion)) { - return; // Skip test if server version is less than required - } - // Test pubsubChannels const regularChannels = await client2.pubsubChannels(); expect(regularChannels).toEqual([regularChannel]); diff --git a/node/tests/ScanTest.test.ts b/node/tests/ScanTest.test.ts index e1ccb9bb06..5c975cacdc 100644 --- a/node/tests/ScanTest.test.ts +++ b/node/tests/ScanTest.test.ts @@ -40,7 +40,7 @@ describe("Scan GlideClusterClient", () => { ) : // setting replicaCount to 1 to facilitate tests routed to replicas await ValkeyCluster.createCluster(true, 3, 1, getServerVersion); - }, 20000); + }, 40000); afterEach(async () => { await flushAndCloseClient(true, cluster.getAddresses(), client); @@ -49,6 +49,8 @@ describe("Scan GlideClusterClient", () => { afterAll(async () => { if (testsFailed === 0) { await cluster.close(); + } else { + await cluster.close(true); } }); @@ -401,6 +403,8 @@ describe("Scan GlideClient", () => { afterAll(async () => { if (testsFailed === 0) { await cluster.close(); + } else { + await cluster.close(true); } }); diff --git a/node/tests/SharedTests.ts b/node/tests/SharedTests.ts index 7b1af14ed1..92e07c30ce 100644 --- a/node/tests/SharedTests.ts +++ b/node/tests/SharedTests.ts @@ -1656,7 +1656,6 @@ export function runBaseTests(config: { await expect(client.hscan(key1, "-1")).rejects.toThrow( RequestError, ); - await expect(client.sscan(key1, "-1")).rejects.toThrow( RequestError, ); @@ -6435,7 +6434,7 @@ export function runBaseTests(config: { [key3], cluster.checkIfServerVersionLessThan("6.0.0") ? 1.0 - : 0.001, + : 0.01, ), ).toBeNull(); @@ -6485,7 +6484,7 @@ export function runBaseTests(config: { [key3], cluster.checkIfServerVersionLessThan("6.0.0") ? 1.0 - : 0.001, + : 0.01, ), ).toBeNull(); @@ -7526,148 +7525,126 @@ export function runBaseTests(config: { it.each([ProtocolVersion.RESP2, ProtocolVersion.RESP3])( `xinfo stream xinfosream test_%p`, async (protocol) => { - await runTest(async (client: BaseClient) => { - const key = uuidv4(); - const groupName = `group-${uuidv4()}`; - const consumerName = `consumer-${uuidv4()}`; - const streamId0_0 = "0-0"; - const streamId1_0 = "1-0"; - const streamId1_1 = "1-1"; + await runTest( + async (client: BaseClient, cluster: ValkeyCluster) => { + const key = uuidv4(); + const groupName = `group-${uuidv4()}`; + const consumerName = `consumer-${uuidv4()}`; + const streamId0_0 = "0-0"; + const streamId1_0 = "1-0"; + const streamId1_1 = "1-1"; - // Setup: add stream entry, create consumer group and consumer, read from stream with consumer - expect( - await client.xadd( - key, - [ - ["a", "b"], - ["c", "d"], - ], - { id: streamId1_0 }, - ), - ).toEqual(streamId1_0); + expect( + await client.xadd( + key, + [ + ["a", "b"], + ["c", "d"], + ], + { id: streamId1_0 }, + ), + ).toEqual(streamId1_0); - expect( - await client.xgroupCreate(key, groupName, streamId0_0), - ).toEqual("OK"); + expect( + await client.xgroupCreate(key, groupName, streamId0_0), + ).toEqual("OK"); - await client.xreadgroup(groupName, consumerName, { - [key]: ">", - }); + await client.xreadgroup(groupName, consumerName, { + [key]: ">", + }); - // test xinfoStream base (non-full) case: - const result = (await client.xinfoStream(key)) as { - length: number; - "radix-tree-keys": number; - "radix-tree-nodes": number; - "last-generated-id": string; - "max-deleted-entry-id": string; - "entries-added": number; - "recorded-first-entry-id": string; - "first-entry": (string | number | string[])[]; - "last-entry": (string | number | string[])[]; - groups: number; - }; + const result = (await client.xinfoStream(key)) as { + length: number; + "radix-tree-keys": number; + "radix-tree-nodes": number; + "last-generated-id": string; + "max-deleted-entry-id": string; + "entries-added": number; + "recorded-first-entry-id": string; + "first-entry": (string | number | string[])[]; + "last-entry": (string | number | string[])[]; + groups: number; + }; - // verify result: - expect(result.length).toEqual(1); - const expectedFirstEntry = ["1-0", ["a", "b", "c", "d"]]; - expect(result["first-entry"]).toEqual(expectedFirstEntry); - expect(result["last-entry"]).toEqual(expectedFirstEntry); - expect(result.groups).toEqual(1); + expect(result.length).toEqual(1); + const expectedFirstEntry = ["1-0", ["a", "b", "c", "d"]]; + expect(result["first-entry"]).toEqual(expectedFirstEntry); + expect(result["last-entry"]).toEqual(expectedFirstEntry); + expect(result.groups).toEqual(1); - // Add one more entry - expect( - await client.xadd(key, [["foo", "bar"]], { - id: streamId1_1, - }), - ).toEqual(streamId1_1); - const fullResult = (await client.xinfoStream(Buffer.from(key), { - fullOptions: 1, - })) as { - length: number; - "radix-tree-keys": number; - "radix-tree-nodes": number; - "last-generated-id": string; - "max-deleted-entry-id": string; - "entries-added": number; - "recorded-first-entry-id": string; - entries: (string | number | string[])[][]; - groups: [ + expect( + await client.xadd(key, [["foo", "bar"]], { + id: streamId1_1, + }), + ).toEqual(streamId1_1); + const fullResult = (await client.xinfoStream( + Buffer.from(key), { - name: string; - "last-delivered-id": string; - "entries-read": number; - lag: number; - "pel-count": number; - pending: (string | number)[][]; - consumers: [ - { - name: string; - "seen-time": number; - "active-time": number; - "pel-count": number; - pending: (string | number)[][]; - }, - ]; + fullOptions: 1, }, - ]; - }; + )) as { + length: number; + "radix-tree-keys": number; + "radix-tree-nodes": number; + "last-generated-id": string; + "max-deleted-entry-id": string; + "entries-added": number; + "recorded-first-entry-id": string; + entries: (string | number | string[])[][]; + groups: [ + { + name: string; + "last-delivered-id": string; + "entries-read": number; + lag: number; + "pel-count": number; + pending: (string | number)[][]; + consumers: [ + { + name: string; + "seen-time": number; + "active-time": number; + "pel-count": number; + pending: (string | number)[][]; + }, + ]; + }, + ]; + }; - // verify full result like: - // { - // length: 2, - // 'radix-tree-keys': 1, - // 'radix-tree-nodes': 2, - // 'last-generated-id': '1-1', - // 'max-deleted-entry-id': '0-0', - // 'entries-added': 2, - // 'recorded-first-entry-id': '1-0', - // entries: [ [ '1-0', ['a', 'b', ...] ] ], - // groups: [ { - // name: 'group', - // 'last-delivered-id': '1-0', - // 'entries-read': 1, - // lag: 1, - // 'pel-count': 1, - // pending: [ [ '1-0', 'consumer', 1722624726802, 1 ] ], - // consumers: [ { - // name: 'consumer', - // 'seen-time': 1722624726802, - // 'active-time': 1722624726802, - // 'pel-count': 1, - // pending: [ [ '1-0', 'consumer', 1722624726802, 1 ] ], - // } - // ] - // } - // ] - // } - expect(fullResult.length).toEqual(2); - expect(fullResult["recorded-first-entry-id"]).toEqual( - streamId1_0, - ); - - // Only the first entry will be returned since we passed count: 1 - expect(fullResult.entries).toEqual([expectedFirstEntry]); - - // compare groupName, consumerName, and pending messages from the full info result: - const fullResultGroups = fullResult.groups; - expect(fullResultGroups.length).toEqual(1); - expect(fullResultGroups[0]["name"]).toEqual(groupName); - - const pendingResult = fullResultGroups[0]["pending"]; - expect(pendingResult.length).toEqual(1); - expect(pendingResult[0][0]).toEqual(streamId1_0); - expect(pendingResult[0][1]).toEqual(consumerName); - - const consumersResult = fullResultGroups[0]["consumers"]; - expect(consumersResult.length).toEqual(1); - expect(consumersResult[0]["name"]).toEqual(consumerName); - - const consumerPendingResult = fullResultGroups[0]["pending"]; - expect(consumerPendingResult.length).toEqual(1); - expect(consumerPendingResult[0][0]).toEqual(streamId1_0); - expect(consumerPendingResult[0][1]).toEqual(consumerName); - }, protocol); + expect(fullResult.length).toEqual(2); + + if (cluster.checkIfServerVersionLessThan("7.0.0")) { + expect( + fullResult["max-deleted-entry-id"], + ).toBeUndefined(); + expect(fullResult["entries-added"]).toBeUndefined(); + expect( + fullResult.groups[0]["entries-read"], + ).toBeUndefined(); + expect(fullResult.groups[0]["lag"]).toBeUndefined(); + } else if (cluster.checkIfServerVersionLessThan("7.2.0")) { + expect(fullResult["recorded-first-entry-id"]).toEqual( + streamId1_0, + ); + + expect( + fullResult.groups[0].consumers[0]["active-time"], + ).toBeUndefined(); + expect( + fullResult.groups[0].consumers[0]["seen-time"], + ).toBeDefined(); + } else { + expect( + fullResult.groups[0].consumers[0]["active-time"], + ).toBeDefined(); + expect( + fullResult.groups[0].consumers[0]["seen-time"], + ).toBeDefined(); + } + }, + protocol, + ); }, config.timeout, ); @@ -10660,7 +10637,7 @@ export function runBaseTests(config: { expect(result[0].pending).toEqual(1); expect(result[0].idle).toBeGreaterThan(0); - if (cluster.checkIfServerVersionLessThan("7.2.0")) { + if (!cluster.checkIfServerVersionLessThan("7.2.0")) { expect(result[0].inactive).toBeGreaterThan(0); } diff --git a/node/tests/TestUtilities.ts b/node/tests/TestUtilities.ts index 0b64b31a04..4bea91dd83 100644 --- a/node/tests/TestUtilities.ts +++ b/node/tests/TestUtilities.ts @@ -600,86 +600,42 @@ export async function encodableTransactionTest( return responseData; } -/** - * Populates a transaction with commands to test the decoded response. - * @param baseTransaction - A transaction. - * @returns Array of tuples, where first element is a test name/description, second - expected return value. - */ -export async function encodedTransactionTest( - baseTransaction: Transaction | ClusterTransaction, -): Promise<[string, GlideReturnType][]> { - const key1 = "{key}" + uuidv4(); // string - const key2 = "{key}" + uuidv4(); // string - const key = "dumpKey"; - const dumpResult = Buffer.from([ - 0, 5, 118, 97, 108, 117, 101, 11, 0, 232, 41, 124, 75, 60, 53, 114, 231, - ]); - const value = "value"; - const valueEncoded = Buffer.from(value); - // array of tuples - first element is test name/description, second - expected return value - const responseData: [string, GlideReturnType][] = []; - - baseTransaction.set(key1, value); - responseData.push(["set(key1, value)", "OK"]); - baseTransaction.set(key2, value); - responseData.push(["set(key2, value)", "OK"]); - baseTransaction.get(key1); - responseData.push(["get(key1)", valueEncoded]); - baseTransaction.get(key2); - responseData.push(["get(key2)", valueEncoded]); - - baseTransaction.set(key, value); - responseData.push(["set(key, value)", "OK"]); - baseTransaction.customCommand(["DUMP", key]); - responseData.push(['customCommand(["DUMP", key])', dumpResult]); - baseTransaction.del([key]); - responseData.push(["del(key)", 1]); - baseTransaction.get(key); - responseData.push(["get(key)", null]); - baseTransaction.customCommand(["RESTORE", key, "0", dumpResult]); - responseData.push([ - 'customCommand(["RESTORE", key, "0", dumpResult])', - "OK", - ]); - baseTransaction.get(key); - responseData.push(["get(key)", valueEncoded]); - - return responseData; -} - /** Populates a transaction with dump and restore commands * * @param baseTransaction - A transaction * @param valueResponse - Represents the encoded response of "value" to compare * @returns Array of tuples, where first element is a test name/description, second - expected return value. */ -export async function DumpAndRestureTest( +export async function DumpAndRestoreTest( baseTransaction: Transaction, - valueResponse: GlideString, + dumpValue: Buffer | null, ): Promise<[string, GlideReturnType][]> { - const key = "dumpKey"; - const dumpResult = Buffer.from([ - 0, 5, 118, 97, 108, 117, 101, 11, 0, 232, 41, 124, 75, 60, 53, 114, 231, - ]); - const value = "value"; + if (dumpValue == null) { + throw new Error("dumpValue is null"); + } + + const key = "{key}-" + uuidv4(); // string + const buffValue = Buffer.from("value"); // array of tuples - first element is test name/description, second - expected return value const responseData: [string, GlideReturnType][] = []; - baseTransaction.set(key, value); - responseData.push(["set(key, value)", "OK"]); + baseTransaction.set(key, "value"); + responseData.push(["set(key, stringValue)", "OK"]); baseTransaction.customCommand(["DUMP", key]); - responseData.push(['customCommand(["DUMP", key])', dumpResult]); + responseData.push(['customCommand(["DUMP", key])', dumpValue]); + baseTransaction.get(key); + responseData.push(["get(key)", buffValue]); baseTransaction.del([key]); responseData.push(["del(key)", 1]); baseTransaction.get(key); responseData.push(["get(key)", null]); - baseTransaction.customCommand(["RESTORE", key, "0", dumpResult]); + baseTransaction.customCommand(["RESTORE", key, "0", dumpValue]); responseData.push([ - 'customCommand(["RESTORE", key, "0", dumpResult])', + 'customCommand(["RESTORE", buffKey, "0", stringValue])', "OK", ]); baseTransaction.get(key); - responseData.push(["get(key)", valueResponse]); + responseData.push(["get(key)", buffValue]); return responseData; } @@ -874,20 +830,20 @@ export async function transactionTest( ]); responseData.push(["lpush(key5, [1, 2, 3, 4])", 4]); - if (gte("7.0.0", version)) { + if (gte(version, "7.0.0")) { baseTransaction.lpush(key24, [field + "1", field + "2"]); responseData.push(["lpush(key22, [1, 2])", 2]); baseTransaction.lmpop([key24], ListDirection.LEFT); responseData.push([ "lmpop([key22], ListDirection.LEFT)", - { [key24]: [field + "2"] }, + [{ key: key24, value: [field + "2"] }], ]); baseTransaction.lpush(key24, [field + "2"]); responseData.push(["lpush(key22, [2])", 2]); baseTransaction.blmpop([key24], ListDirection.LEFT, 0.1, 1); responseData.push([ "blmpop([key22], ListDirection.LEFT, 0.1, 1)", - { [key24]: [field + "2"] }, + [{ key: key24, value: [field + "2"] }], ]); } @@ -1490,31 +1446,31 @@ export async function transactionTest( baseTransaction.geoadd( key18, new Map([ - ["Palermo", { longitude: 13.361389, latitude: 38.115556 }], - ["Catania", { longitude: 15.087269, latitude: 37.502669 }], + ["palermo", { longitude: 13.361389, latitude: 38.115556 }], + ["catania", { longitude: 15.087269, latitude: 37.502669 }], ]), ); - responseData.push(["geoadd(key18, { Palermo: ..., Catania: ... })", 2]); - baseTransaction.geopos(key18, ["Palermo", "Catania"]); + responseData.push(["geoadd(key18, { palermo: ..., catania: ... })", 2]); + baseTransaction.geopos(key18, ["palermo", "catania"]); responseData.push([ - 'geopos(key18, ["Palermo", "Catania"])', + 'geopos(key18, ["palermo", "catania"])', [ [13.36138933897018433, 38.11555639549629859], [15.08726745843887329, 37.50266842333162032], ], ]); - baseTransaction.geodist(key18, "Palermo", "Catania"); - responseData.push(['geodist(key18, "Palermo", "Catania")', 166274.1516]); - baseTransaction.geodist(key18, "Palermo", "Catania", { + baseTransaction.geodist(key18, "palermo", "catania"); + responseData.push(['geodist(key18, "palermo", "catania")', 166274.1516]); + baseTransaction.geodist(key18, "palermo", "catania", { unit: GeoUnit.KILOMETERS, }); responseData.push([ - 'geodist(key18, "Palermo", "Catania", { unit: GeoUnit.KILOMETERS })', + 'geodist(key18, "palermo", "catania", { unit: GeoUnit.KILOMETERS })', 166.2742, ]); - baseTransaction.geohash(key18, ["Palermo", "Catania", "NonExisting"]); + baseTransaction.geohash(key18, ["palermo", "catania", "NonExisting"]); responseData.push([ - 'geohash(key18, ["Palermo", "Catania", "NonExisting"])', + 'geohash(key18, ["palermo", "catania", "NonExisting"])', ["sqc8b49rny0", "sqdtr74hyu0", null], ]); baseTransaction.zadd(key23, { one: 1.0 }); @@ -1533,7 +1489,7 @@ export async function transactionTest( baseTransaction .geosearch( key18, - { member: "Palermo" }, + { member: "palermo" }, { radius: 200, unit: GeoUnit.KILOMETERS }, { sortOrder: SortOrder.ASC }, ) @@ -1544,7 +1500,7 @@ export async function transactionTest( ) .geosearch( key18, - { member: "Palermo" }, + { member: "palermo" }, { radius: 200, unit: GeoUnit.KILOMETERS }, { sortOrder: SortOrder.ASC, @@ -1567,18 +1523,18 @@ export async function transactionTest( }, ); responseData.push([ - 'geosearch(key18, "Palermo", R200 KM, ASC)', - ["Palermo", "Catania"], + 'geosearch(key18, "palermo", R200 KM, ASC)', + ["palermo", "catania"], ]); responseData.push([ "geosearch(key18, (15, 37), 400x400 KM, ASC)", - ["Palermo", "Catania"], + ["palermo", "catania"], ]); responseData.push([ - 'geosearch(key18, "Palermo", R200 KM, ASC 2 3x true)', + 'geosearch(key18, "palermo", R200 KM, ASC 2 3x true)', [ [ - "Palermo", + "palermo", [ 0.0, 3479099956230698, @@ -1586,7 +1542,7 @@ export async function transactionTest( ], ], [ - "Catania", + "catania", [ 166.2742, 3479447370796909, @@ -1599,7 +1555,7 @@ export async function transactionTest( "geosearch(key18, (15, 37), 400x400 KM, ASC 2 3x true)", [ [ - "Catania", + "catania", [ 56.4413, 3479447370796909, @@ -1607,7 +1563,7 @@ export async function transactionTest( ], ], [ - "Palermo", + "palermo", [ 190.4424, 3479099956230698, @@ -1758,7 +1714,8 @@ export async function transactionTest( } baseTransaction.wait(1, 200); - responseData.push(["wait(1, 200)", 1]); + if (gte(version, "7.0.0")) responseData.push(["wait(1, 200)", 1]); + else responseData.push(["wait(1, 200)", 0]); return responseData; } diff --git a/node/tests/setup.js b/node/tests/setup.js new file mode 100644 index 0000000000..e8aa09f2bf --- /dev/null +++ b/node/tests/setup.js @@ -0,0 +1,7 @@ +import { beforeAll } from "@jest/globals"; +import { Logger } from ".."; + +beforeAll(() => { + Logger.init("error"); + Logger.setLoggerConfig("error"); +}); diff --git a/python/Cargo.toml b/python/Cargo.toml index 3945322cd2..16ec93f10c 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -11,11 +11,20 @@ name = "glide" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "^0.20", features = ["extension-module", "num-bigint"] } +pyo3 = { version = "^0.22", features = [ + "extension-module", + "num-bigint", + "gil-refs", +] } bytes = { version = "1.6.0" } -redis = { path = "../glide-core/redis-rs/redis", features = ["aio", "tokio-comp", "connection-manager","tokio-rustls-comp"] } +redis = { path = "../glide-core/redis-rs/redis", features = [ + "aio", + "tokio-comp", + "connection-manager", + "tokio-rustls-comp", +] } glide-core = { path = "../glide-core", features = ["socket-layer"] } -logger_core = {path = "../logger_core"} +logger_core = { path = "../logger_core" } [package.metadata.maturin] python-source = "python" diff --git a/python/src/lib.rs b/python/src/lib.rs index 994c7f7b4e..d00034e8cf 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -15,7 +15,7 @@ pub const DEFAULT_TIMEOUT_IN_MILLISECONDS: u32 = glide_core::client::DEFAULT_RESPONSE_TIMEOUT.as_millis() as u32; pub const MAX_REQUEST_ARGS_LEN: u32 = MAX_REQUEST_ARGS_LENGTH as u32; -#[pyclass] +#[pyclass(eq, eq_int)] #[derive(PartialEq, Eq, PartialOrd, Clone)] pub enum Level { Error = 0, @@ -48,6 +48,7 @@ pub struct ClusterScanCursor { #[pymethods] impl ClusterScanCursor { #[new] + #[pyo3(signature = (new_cursor=None))] fn new(new_cursor: Option) -> Self { match new_cursor { Some(cursor) => ClusterScanCursor { cursor }, @@ -78,7 +79,7 @@ pub struct Script { #[pymethods] impl Script { #[new] - fn new(code: &PyAny) -> PyResult { + fn new(code: &Bound) -> PyResult { let hash = if let Ok(code_str) = code.extract::() { glide_core::scripts_container::add_script(code_str.as_bytes()) } else if let Ok(code_bytes) = code.extract::<&PyBytes>() { @@ -103,7 +104,7 @@ impl Script { /// A Python module implemented in Rust. #[pymodule] -fn glide(_py: Python, m: &PyModule) -> PyResult<()> { +fn glide(_py: Python, m: &Bound) -> PyResult<()> { m.add_class::()?; m.add_class::