From f227a32476c6272a45a14a5d68ed330e35f0e9a5 Mon Sep 17 00:00:00 2001 From: avifenesh Date: Tue, 22 Oct 2024 19:46:19 +0000 Subject: [PATCH] fixed triggers Signed-off-by: avifenesh --- .github/DEVELOPER.md | 91 ++++---- .github/json_matrices/build-matrix.json | 30 ++- .../supported-languages-versions.json | 12 +- .../workflows/create-test-matrices/action.yml | 11 +- .github/workflows/csharp.yml | 161 +++++++++---- .github/workflows/go.yml | 221 ++++++++++-------- .github/workflows/java.yml | 100 ++++---- .github/workflows/nightly.yml | 90 +++++++ .github/workflows/node.yml | 140 ++++++----- .github/workflows/npm-cd.yml | 3 +- .github/workflows/pypi-cd.yml | 4 +- .github/workflows/python.yml | 133 +++++++---- .github/workflows/rust.yml | 55 +++-- 13 files changed, 674 insertions(+), 377 deletions(-) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md index 710bd08dca..70798b4077 100644 --- a/.github/DEVELOPER.md +++ b/.github/DEVELOPER.md @@ -1,17 +1,25 @@ # CI/CD Workflow Guide -TODO: Add a description of the CI/CD workflow and its components. - ### Overview Our CI/CD pipeline tests and builds our project across multiple languages, versions, and environments. This guide outlines the key components and processes of our workflow. ### Workflow Triggers -- Push to `main` branch -- Pull requests -- Scheduled runs (daily) -- Manual trigger (workflow_dispatch) +* Pull requests +* Pushes to `main` or release branches (PR merges) +* Scheduled runs (daily) - starts CI pipelines for all clients +* Manual trigger (`workflow_dispatch`) - a developer can start a client's pipeline or the scheduled one to run all pipelines on demand + +Job triggers + +### Test coverage + +There are two levels of testing: the basic one and full (_aka_ `full-matrix`). +Basic amount of testing is executed on every open and merged PR. The full set of tests is executed by the scheduled job. +A developer can select the level when starting a job, either scheduled or client's pipeline. + +Matrices ### Language-Specific Workflows @@ -21,12 +29,12 @@ Each language has its own workflow file with similar structure but language-spec #### Matrix Files -While workflows are language-specific, the matrix files are shared across all workflows. +While workflows are language-specific, the matrix files are shared across all workflows. Workflows are starting by loading the matrix files from the `.github/json_matrices` directory. -- `engine-matrix.json`: Defines the versions of Valkey engine to test against. -- `build-matrix.json`: Defines the host environments for testing. -- `supported-languages-version.json`: Defines the supported versions of languages. +* `engine-matrix.json`: Defines the versions of the engine to test against. +* `build-matrix.json`: Defines the host environments for testing. +* `supported-languages-versions.json`: Defines the supported versions of languages. All matrices have a `run` like field which specifies if the configuration should be tested on every workflow run. This allows for flexible control over which configurations are tested in different scenarios, optimizing CI/CD performance and resource usage. @@ -36,12 +44,16 @@ This allows for flexible control over which configurations are tested in differe Defines the versions of Valkey engine to test against: ```json -[{ "type": "valkey", "version": "7.2.5", "run": "always" }] +[ + { "type": "valkey", "version": "7.2.5", "run": "always" }, + // ... other configurations +] ``` -- `type`: The type of engine (e.g., Valkey, Redis). -- `version`: The version of the engine. -- `run`: Specifies if the engine version should be tested on every workflow. +* `type`: The type of engine (e.g., Valkey, Redis). +* `version`: Specifies the engine version that the workflow should checkout. + For example, "7.2.5" represents a release tag, while "7.0" denotes a branch name. The workflow should use this parameter to checkout the specific release version or branch to build the engine with the appropriate version. +* `run`: Specifies if the engine version should be tested on every workflow. #### Build Matrix (build-matrix.json) @@ -58,11 +70,10 @@ Defines the host environments for testing: // ... other configurations ] ``` - -- `OS`: The operating system of the host. -- `RUNNER`: The GitHub runner to use. -- `TARGET`: The target environment. -- `run`: Specifies which language workflows should use this host configuration, always means run on each workflow trigger. +* `OS`: The operating system of the host. +* `RUNNER`: The GitHub runner to use. +* `TARGET`: The target environment as defined in Rust. To see a list of available targets, run `rustup target list`. +* `run`: Specifies which language workflows should use this host configuration. The value `always` indicates that the configuration should be used for every workflow trigger. #### Supported Languages Version (supported-languages-version.json) @@ -74,38 +85,37 @@ Defines the supported versions of languages: "language": "java", "versions": ["11", "17"], "always-run-versions": ["17"] - } + }, // ... other configurations ] ``` - -- `language`: The language for which the version is supported. -- `versions`: The full versions supported of the language which will test against scheduled. -- `always-run-versions`: The versions which will be tested in every workflow run. +* `language`: The language for which the version is supported. +* `versions`: The full versions supported of the language which will test against scheduled. +* `always-run-versions`: The versions that will always be tested, regardless of the workflow trigger. #### Triggering Workflows -Push to main or create a pull request to run workflows automatically. -Use workflow_dispatch for manual triggers, accepting inputs of full-matrix which is a boolean value to run all configurations. -Scheduled runs are triggered daily to ensure regular testing of all configurations. +- Push to `main` by merging a PR or create a new pull request to run workflows automatically. +- Use `workflow_dispatch` for manual triggers, accepting a boolean configuration parameter to run all configurations. +- Scheduled runs are triggered daily to ensure regular testing of all configurations. ### Mutual vs. Language-Specific Components #### Mutual -`Matrix files` - `.github/json_matrices` -`Shared dependencies installation` - `.github/workflows/install-shared-dependencies/action.yml` -`Linting Rust` - `.github/workflows/lint-rust/action.yml` +- Matrix files - `.github/json_matrices/` +- Shared dependencies installation - `.github/workflows/install-shared-dependencies/action.yml` +- Rust linters - `.github/workflows/lint-rust/action.yml` #### Language-Specific -`Package manager commands` -`Testing frameworks` -`Build processes` +- Package manager commands +- Testing frameworks +- Build processes ### Customizing Workflows -Modify `[language].yml` files to adjust language-specific steps. +Modify `.yml` files to adjust language-specific steps. Update matrix files to change tested versions or environments. Adjust cron schedules in workflow files for different timing of scheduled runs. @@ -119,14 +129,13 @@ We use dynamic matrices for our CI/CD workflows, which are created using the `cr 2. It reads the `engine-matrix.json`, `build-matrix.json`, and `supported-languages-version.json` files. 3. It filters the matrices based on the inputs and the event type. 4. It generates three matrices: - - Engine matrix: Defines the types and versions of the engine to test against, for example Valkey 7.2.5. - - Host matrix: Defines the host platforms to run the tests on, for example Ubuntu on ARM64. - - Language-version matrix: Defines the supported versions of languages, for example python 3.8. + - Engine matrix: Defines the types and versions of the engine to test against, for example Valkey 7.2.5. + - Host matrix: Defines the host platforms to run the tests on, for example Ubuntu on ARM64. + - Language-version matrix: Defines the supported versions of languages, for example python 3.8. #### Outputs - -- `engine-matrix-output`: The generated engine matrix. -- `host-matrix-output`: The generated host matrix. -- `language-version-matrix-output`: The generated language version matrix. +- `engine-matrix-output`: The generated engine matrix. +- `host-matrix-output`: The generated host matrix. +- `language-version-matrix-output`: The generated language version matrix. This dynamic matrix generation allows for flexible and efficient CI/CD workflows, adapting the test configurations based on the type of change and the specific language being tested. diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index f776ad3c64..37217013ec 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -5,8 +5,9 @@ "RUNNER": "ubuntu-latest", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", - "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["always", "python", "node", "java"] + "PACKAGE_MANAGERS": ["pypi", "npm", "maven"], + "run": "always", + "languages": ["python", "node", "java", "go", "dotnet"] }, { "OS": "ubuntu", @@ -14,9 +15,9 @@ "RUNNER": ["self-hosted", "Linux", "ARM64"], "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-gnu", - "PACKAGE_MANAGERS": ["pypi", "npm"], + "PACKAGE_MANAGERS": ["pypi", "npm", "maven"], "CONTAINER": "2_28", - "run": ["python", "node", "java"] + "languages": ["python", "node", "java", "go", "dotnet"] }, { "OS": "macos", @@ -24,8 +25,8 @@ "RUNNER": "macos-12", "ARCH": "x64", "TARGET": "x86_64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["python", "node", "java"] + "PACKAGE_MANAGERS": ["pypi", "npm", "maven"], + "languages": ["python", "node", "java", "go", "dotnet"] }, { "OS": "macos", @@ -33,8 +34,8 @@ "RUNNER": "macos-latest", "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["python", "node", "java"] + "PACKAGE_MANAGERS": ["pypi", "npm", "maven"], + "languages": ["python", "node", "java", "go", "dotnet"] }, { "OS": "ubuntu", @@ -45,7 +46,7 @@ "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged --rm", "PACKAGE_MANAGERS": ["npm"], - "run": ["node"] + "languages": ["node"] }, { "OS": "ubuntu", @@ -56,6 +57,15 @@ "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged", "PACKAGE_MANAGERS": ["npm"], - "run": ["node"] + "languages": ["node"] + }, + { + "OS": "amazon-linux", + "NAMED_OS": "linux", + "RUNNER": "ubuntu-latest", + "ARCH": "x64", + "TARGET": "x86_64-unknown-linux-gnu", + "IMAGE": "amazonlinux:latest", + "languages": ["python", "node", "java", "go", "dotnet"] } ] diff --git a/.github/json_matrices/supported-languages-versions.json b/.github/json_matrices/supported-languages-versions.json index bc6fd5f472..42808eb0d6 100644 --- a/.github/json_matrices/supported-languages-versions.json +++ b/.github/json_matrices/supported-languages-versions.json @@ -11,7 +11,17 @@ }, { "language": "node", - "full-versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], + "versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], "always-run-versions": ["16.x", "20.x"] + }, + { + "language": "dotnet", + "versions": ["8.0", "6.0"], + "always-run-versions": ["8.0"] + }, + { + "language": "go", + "versions": ["1.22.0", "1.18.10"], + "always-run-versions": ["1.22.0"] } ] diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml index ce03c014a3..1cb49d92e3 100644 --- a/.github/workflows/create-test-matrices/action.yml +++ b/.github/workflows/create-test-matrices/action.yml @@ -13,11 +13,6 @@ inputs: description: "Run the full matrix" required: true type: boolean - containers: - # if set to `true`, "load-host-matrix" step selects blocks with "IMAGE" only from `build-matrix` file, otherwise they are ignored - description: "`true` to pick containers to the host list instead of runners" - default: false - type: boolean outputs: engine-matrix-output: @@ -53,7 +48,7 @@ runs: shell: bash run: | set -o pipefail - [[ ${{ inputs.containers }} == true ]] && CONDITION=".IMAGE?" || CONDITION=".IMAGE == null" + [[ "${{ inputs.containers }}" == "true" ]] && CONDITION=".IMAGE?" || CONDITION=".IMAGE == null" echo 'Select runners (VMs) to run tests on' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.run-full-matrix }}" == "false" ]]; then echo 'Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix' @@ -72,9 +67,9 @@ runs: echo 'Select language (framework/SDK) versions to run tests on' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.run-full-matrix }}" == "false" ]]; then echo 'Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix' - jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"]][0] // []' < .github/json_matrices/supported-language-versions.json | awk '{ printf "version-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT + jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"]][0] // []' < .github/json_matrices/supported-languages-versions.json | awk '{ printf "version-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT else echo 'Pick language versions listed in 'versions' - on cron (schedule) or if manually triggered job requires a full matrix' - jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .versions][0]' < .github/json_matrices/supported-language-versions.json | awk '{ printf "version-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT + jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .versions][0]' < .github/json_matrices/supported-languages-versions.json | awk '{ printf "version-matrix=%s\n", $1 }' | tee -a $GITHUB_OUTPUT fi cat $GITHUB_OUTPUT diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index fa1bb6d4e3..585378fdb2 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -6,69 +6,78 @@ on: paths: - csharp/** - glide-core/src/** + - utils/cluster_manager.py - submodules/** - .github/workflows/csharp.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** pull_request: paths: - csharp/** - glide-core/src/** - submodules/** + - utils/cluster_manager.py - .github/workflows/csharp.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** workflow_dispatch: + inputs: + full-matrix: + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: + required: false + type: string + description: "(Optional) Test run name" + + workflow_call: permissions: contents: read concurrency: - group: C#-${{ github.head_ref || github.ref }} + group: C#-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + +env: + CARGO_TERM_COLOR: always + jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: dotnet + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} - run-tests: - needs: load-engine-matrix - timeout-minutes: 25 + test-csharp: + needs: get-matrices + timeout-minutes: 35 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - dotnet: - # - '6.0' - - "8.0" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu, - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + dotnet: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -89,12 +98,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} - - name: Format - working-directory: ./csharp - run: dotnet format --verify-no-changes --verbosity diagnostic - - name: Test dotnet ${{ matrix.dotnet }} - working-directory: ./csharp + working-directory: csharp run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror - uses: ./.github/workflows/test-benchmark @@ -106,15 +111,84 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-reports-dotnet-${{ matrix.dotnet }}-redis-${{ matrix.redis }}-${{ matrix.host.RUNNER }} + name: test-reports-dotnet-${{ matrix.dotnet }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} path: | csharp/TestReport.html benchmarks/results/* utils/clusters/** - # TODO Add amazonlinux + get-containers: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: dotnet + run-full-matrix: true + containers: true + + test-csharp-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + # Don't use generated matrix for dotnet until net6.0 compatibility issues resolved on amazon linux + # dotnet: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + dotnet: ["8.0"] + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options + steps: + - name: Install git + run: | + yum update + yum install -y git tar findutils libicu + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` + - uses: actions/checkout@v4 + with: + submodules: recursive - lint-rust: + - name: Set up dotnet ${{ matrix.dotnet }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.dotnet }} + + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test dotnet ${{ matrix.dotnet }} + working-directory: csharp + run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: test-reports-dotnet-${{ matrix.dotnet }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} + path: | + csharp/TestReport.html + benchmarks/results/* + utils/clusters/** + + lint: timeout-minutes: 10 runs-on: ubuntu-latest steps: @@ -122,7 +196,12 @@ jobs: with: submodules: recursive - - uses: ./.github/workflows/lint-rust + - name: lint rust + uses: ./.github/workflows/lint-rust with: - cargo-toml-folder: ./csharp/lib + cargo-toml-folder: csharp/lib github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Format + working-directory: csharp + run: dotnet format --verify-no-changes --verbosity diagnostic diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 69254a8cd5..15b55444a2 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -6,66 +6,75 @@ on: paths: - glide-core/src/** - submodules/** + - utils/cluster_manager.py - go/** - .github/workflows/go.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** pull_request: paths: - glide-core/src/** - submodules/** + - utils/cluster_manager.py - go/** - .github/workflows/go.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** workflow_dispatch: - + inputs: + full-matrix: + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: + required: false + type: string + description: "(Optional) Test run name" + + workflow_call: + concurrency: - group: go-${{ github.head_ref || github.ref }} + group: go-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + +env: + CARGO_TERM_COLOR: always + jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: go + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} - build-and-test-go-client: - needs: load-engine-matrix + test-go: + needs: get-matrices timeout-minutes: 35 strategy: - # Run all jobs fail-fast: false matrix: - go: - - "1.22.0" - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu, - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + go: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -99,14 +108,9 @@ jobs: working-directory: ./go run: make build - - name: Run linters - working-directory: ./go - run: make lint-ci - - name: Run tests working-directory: ./go - run: | - make test + run: make test - uses: ./.github/workflows/test-benchmark with: @@ -117,94 +121,111 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: reports-go-${{ matrix.go }}-redis-${{ matrix.redis }}-${{ matrix.os }} + name: test-report-go-${{ matrix.go }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} path: | utils/clusters/** benchmarks/results/** - build-amazonlinux-latest: - if: github.repository_owner == 'valkey-io' + lint: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: ./.github/workflows/lint-rust + with: + cargo-toml-folder: go + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v5 + with: + go-version: "1.22.0" + cache-dependency-path: go/go.sum + + - name: Install protoc + uses: ./.github/workflows/install-rust-and-protoc + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install and run linters + working-directory: go + run: | + sudo apt install golang-goprotobuf-dev + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/ + make install-dev-tools install-build-tools generate-protobuf generate-bindings lint-ci + + get-containers: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: go + run-full-matrix: true + containers: true + + test-go-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 strategy: - # Run all jobs fail-fast: false matrix: - go: - - 1.22.0 - runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 15 + go: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options steps: - name: Install git run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm yum update - yum install -y git - git --version + yum install -y git tar + git config --global --add safe.directory "$GITHUB_WORKSPACE" + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + cache-dependency-path: go/go.sum - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: Install Go - run: | - yum -y install wget - yum -y install tar - wget https://go.dev/dl/go${{ matrix.go }}.linux-amd64.tar.gz - tar -C /usr/local -xzf go${{ matrix.go }}.linux-amd64.tar.gz - echo "/usr/local/go/bin" >> $GITHUB_PATH - echo "$HOME/go/bin" >> $GITHUB_PATH - - - name: Install tools for Go ${{ matrix.go }} - working-directory: ./go - run: make install-tools-go${{ matrix.go }} - - - name: Set LD_LIBRARY_PATH - run: echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/" >> $GITHUB_ENV - - - name: Build client - working-directory: ./go - run: make build - - - name: Run linters - working-directory: ./go - run: make lint-ci + engine-version: ${{ matrix.engine.version }} - - name: Run tests - working-directory: ./go + - name: Install & build & test + working-directory: go run: | - make test + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/go/target/release/deps/ + make install-tools-go${{ matrix.go }} build test - - name: Upload cluster manager logs + - name: Upload test reports if: always() continue-on-error: true uses: actions/upload-artifact@v4 with: - name: cluster-manager-logs-${{ matrix.go }}-redis-6-amazonlinux + name: test-reports-go-${{ matrix.go }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | utils/clusters/** - - lint-rust: - timeout-minutes: 15 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: ./.github/workflows/lint-rust - with: - cargo-toml-folder: ./go - name: lint go rust + benchmarks/results/** \ No newline at end of file diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index ed8522a9c0..185dc4d42d 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -7,42 +7,53 @@ on: - glide-core/src/** - submodules/** - java/** + - utils/cluster_manager.py - .github/workflows/java.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + pull_request: paths: - glide-core/src/** - submodules/** - java/** + - utils/cluster_manager.py - .github/workflows/java.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: required: false - default: "false" + type: string + description: "(Optional) Test run name" - schedule: - - cron: "0 0 * * *" + workflow_call: concurrency: - group: java-${{ github.head_ref || github.ref }} + group: java-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -53,18 +64,18 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: java - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} test-java: needs: get-matrices timeout-minutes: 35 strategy: - # Run all jobs fail-fast: false matrix: java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -119,40 +130,52 @@ jobs: benchmarks/results/** java/client/build/reports/spotbugs/** - build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') - needs: get-matrices + get-containers: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: java + run-full-matrix: true + containers: true + + test-java-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 strategy: - # Run all jobs fail-fast: false matrix: - java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} - runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 35 + java: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options steps: - name: Install git run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm yum update - yum install -y git - git --version - + yum install -y git tar java-${{ matrix.java }}-amazon-corretto-devel.x86_64 + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` - uses: actions/checkout@v4 - - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive + with: + submodules: recursive - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} @@ -162,10 +185,6 @@ jobs: version: "26.1" repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install Java - run: | - yum install -y java-${{ matrix.java }}-amazon-corretto-devel.x86_64 - - name: Build java wrapper working-directory: java run: ./gradlew --continue build -x javadoc @@ -175,7 +194,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-reports-${{ matrix.java }}-${{ matrix.engine }}-amazon-linux + name: test-reports-java-${{ matrix.java }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | java/client/build/reports/** java/integTest/build/reports/** @@ -191,7 +210,8 @@ jobs: - uses: ./.github/workflows/lint-rust with: - cargo-toml-folder: ./java + cargo-toml-folder: java + github-token: ${{ secrets.GITHUB_TOKEN }} name: lint java rust test-modules: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..1c38a18f5a --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,90 @@ +name: Nightly tests + +on: + workflow_dispatch: # note: if started manually, it won't run all matrix + inputs: + full-matrix: + description: "Run the full engine and host matrix" + type: boolean + default: false + # GHA supports up to 10 inputs, there is no option for multi-choice + core: + description: "Test GLIDE core" + type: boolean + default: true + node: + description: "Test Node client" + type: boolean + default: true + python: + description: "Test Python client" + type: boolean + default: true + java: + description: "Test Java client" + type: boolean + default: true + csharp: + description: "Test C# client" + type: boolean + default: false + go: + description: "Test Golang client" + type: boolean + default: false + + schedule: # running tests by cron is disabled on forks by the condition defined for each job + - cron: "0 3 * * *" # Runs at 03:00 (3 AM) UTC every day + +concurrency: + group: nightly-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} + cancel-in-progress: true + +# TODO matrix by workflow (`uses`) - not supported yet by GH +jobs: + check-input: + runs-on: ubuntu-latest + steps: + - name: no tests selected + run: false + if: github.event_name == 'workflow_dispatch' && inputs.core == false && inputs.java == false && inputs.python == false && inputs.node == false && inputs.java == false && inputs.csharp == false && inputs.go == false + + run-full-tests-for-core: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.core == true) + uses: ./.github/workflows/rust.yml + name: Run CI for GLIDE core lib + secrets: inherit + + run-full-tests-for-java: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.java == true) + uses: ./.github/workflows/java.yml + name: Run CI for java client + secrets: inherit + + run-full-tests-for-python: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.python == true) + uses: ./.github/workflows/python.yml + name: Run CI for python client + secrets: inherit + + run-full-tests-for-node: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.node == true) + uses: ./.github/workflows/node.yml + name: Run CI for node client + secrets: inherit + + run-full-tests-for-csharp: + # C# deactivated in cron, uncomment condition to activate + #if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.csharp == true) + if: (github.event_name == 'workflow_dispatch' && inputs.csharp == true) + uses: ./.github/workflows/csharp.yml + name: Run CI for c# client + secrets: inherit + + run-full-tests-for-go: + # Go deactivated in cron, uncomment condition to activate + #if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.go == true) + if: (github.event_name == 'workflow_dispatch' && inputs.go == true) + uses: ./.github/workflows/go.yml + name: Run CI for go client + secrets: inherit diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 1ae1973838..3498ac70d7 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -14,7 +14,8 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml pull_request: paths: - glide-core/src/** @@ -27,16 +28,20 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: required: false - default: "false" + type: string + description: "(Optional) Test run name" - schedule: - - cron: "0 1 * * *" + workflow_call: concurrency: group: node-${{ github.head_ref || github.ref }} @@ -45,11 +50,13 @@ concurrency: env: CARGO_TERM_COLOR: always +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -61,10 +68,10 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: node - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} test-node: - runs-on: ubuntu-latest + runs-on: ${{ matrix.host.RUNNER }} needs: [get-matrices] timeout-minutes: 25 strategy: @@ -72,7 +79,7 @@ jobs: matrix: engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} - node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output)}} + node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} steps: - uses: actions/checkout@v4 with: @@ -89,6 +96,8 @@ jobs: uses: ./.github/workflows/build-node-wrapper with: os: ${{ matrix.host.OS }} + named_os: ${{ matrix.host.NAMED_OS }} + arch: ${{ matrix.host.ARCH }} target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} @@ -124,7 +133,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.node }}-ubuntu + name: test-report-node-${{ matrix.node }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.OS }}-${{ matrix.host.ARCH }} path: | node/test-report*.html utils/clusters/** @@ -141,90 +150,79 @@ jobs: - uses: ./.github/workflows/lint-rust with: cargo-toml-folder: ./node/rust-client + github-token: ${{ secrets.GITHUB_TOKEN }} name: lint node rust - build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') + get-containers: runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 15 - steps: - - name: Install git - run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm - yum install -y git - git --version + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + steps: - uses: actions/checkout@v4 - - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive - - - name: Install NodeJS - run: | - yum install -y nodejs - - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: test - run: npm test - working-directory: ./node - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices with: - name: test-report-node-amazonlinux - path: | - node/test-report*.html - utils/clusters/** - benchmarks/results/** + language-name: node + run-full-matrix: true + containers: true - build-and-test-linux-musl-on-x86: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') - name: Build and test Node wrapper on Linux musl - runs-on: ubuntu-latest + test-node-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + node: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} container: - image: node:alpine - options: --user root --privileged - + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options steps: - name: Install git run: | - apk update - apk add git - + if [[ ${{ contains(matrix.host.TARGET, 'musl') }} == true ]]; then + apk update + apk add --no-cache git tar + elif [[ ${{ contains(matrix.host.IMAGE, 'amazonlinux') }} == true ]]; then + yum update + yum install -y git tar + fi + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` - uses: actions/checkout@v4 with: submodules: recursive - name: Setup musl on Linux + if: ${{ contains(matrix.host.TARGET, 'musl') }} uses: ./.github/workflows/setup-musl-on-linux with: workspace: $GITHUB_WORKSPACE npm-scope: ${{ secrets.NPM_SCOPE }} npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} + - name: Setup Node + uses: actions/setup-node@v4 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + with: + node-version: ${{ matrix.node }} + - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: - os: ubuntu - named_os: linux - arch: x64 - target: x86_64-unknown-linux-musl + os: ${{ matrix.host.OS }} + named_os: ${{ matrix.host.NAMED_OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" + engine-version: ${{ matrix.engine.version }} + arch: ${{ matrix.host.ARCH }} - name: test run: npm test @@ -235,7 +233,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-report-node-linux-musl + name: test-report-node-${{ matrix.node }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | node/test-report*.html utils/clusters/** diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index af814b7df1..b3809e7f67 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -11,7 +11,8 @@ on: - .github/workflows/install-rust-and-protoc/action.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml push: tags: - "v*.*" diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index 3805bc014f..3a0b423fa0 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -10,7 +10,8 @@ on: - .github/workflows/start-self-hosted-runner/action.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/json_matrices/** + - .github/workflows/create-test-matrices/action.yml push: tags: - "v*.*" @@ -120,7 +121,6 @@ jobs: brew update brew install python@3.9 - - name: Setup Python for self-hosted Ubuntu runners if: contains(matrix.build.OS, 'ubuntu') && contains(matrix.build.RUNNER, 'self-hosted') run: | diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a90e2d3fcc..ca2086f105 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -14,9 +14,9 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/json_matrices/engine-matrix.json - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** pull_request: paths: @@ -30,21 +30,25 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/json_matrices/engine-matrix.json - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" + type: boolean + default: false + name: required: false - default: "false" + type: string + description: "(Optional) Test run name" - schedule: - - cron: "0 2 * * *" + workflow_call: concurrency: - group: python-${{ github.head_ref || github.ref }} + group: python-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true permissions: @@ -52,22 +56,25 @@ permissions: # Allows the GITHUB_TOKEN to make an API call to generate an OIDC token. id-token: write +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} - version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output}} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - uses: actions/checkout@v4 - id: get-matrices uses: ./.github/workflows/create-test-matrices with: language-name: python - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} test-python: runs-on: ${{ matrix.host.RUNNER }} @@ -76,9 +83,9 @@ jobs: strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} steps: - uses: actions/checkout@v4 with: @@ -135,6 +142,8 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** + + # run pubsub tests in another job - they take too much time test-pubsub-python: runs-on: ${{ matrix.host.RUNNER }} needs: get-matrices @@ -142,8 +151,8 @@ jobs: strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 @@ -190,7 +199,8 @@ jobs: - name: lint rust uses: ./.github/workflows/lint-rust with: - cargo-toml-folder: ./python + cargo-toml-folder: python + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies if: always() @@ -219,53 +229,77 @@ jobs: working-directory: ./python run: | black --check --diff . - build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') + + get-containers: runs-on: ubuntu-latest - container: amazonlinux:latest - timeout-minutes: 15 - steps: - - name: Install git - run: | - yum -y remove git - yum -y remove git-* - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm - yum install -y git - git --version + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + steps: - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: python + run-full-matrix: true + containers: true - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive - - - name: Install python + test-python-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + # Don't use generated matrix for python until compatibility issues resolved on amazon linux + # python: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options + steps: + - name: Install git and python run: | - yum install -y python3 + yum update + yum install -y git tar python3 + python3 -m ensurepip --upgrade + python3 -m pip install --upgrade pip + python3 -m pip install mypy-protobuf virtualenv + echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV + # Replace `:` in the variable otherwise it can't be used in `upload-artifact` + - uses: actions/checkout@v4 + with: + submodules: recursive - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" + engine-version: ${{ matrix.engine.version }} - - name: Test compatibility with pytest + - name: Test with pytest working-directory: ./python run: | source .env/bin/activate - pytest --asyncio-mode=auto -m smoke_test --html=pytest_report.html --self-contained-html + cd python/tests/ + pytest --asyncio-mode=auto --html=pytest_report.html --self-contained-html - name: Upload test reports if: always() continue-on-error: true uses: actions/upload-artifact@v4 with: - name: smoke-test-report-amazon-linux + name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }} path: | python/python/tests/pytest_report.html + utils/clusters/** + benchmarks/results/** start-self-hosted-runner: if: github.event.pull_request.head.repo.owner.login == 'valkey-io' @@ -290,9 +324,16 @@ jobs: strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} - python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + python: + - "3.12" + host: + - { + OS: "ubuntu", + NAMED_OS: "linux", + RUNNER: ["self-hosted", "Linux", "ARM64"], + TARGET: "aarch64-unknown-linux-gnu", + } steps: - name: Setup self-hosted runner access @@ -329,6 +370,6 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: modules-test-report-${{ matrix.host.TARGET }}-python-${{ matrix.python }}-server-${{ matrix.engine.version }} + name: modules-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} path: | python/python/tests/pytest_report.html diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 09769b732e..6c28b33a15 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -12,7 +12,8 @@ on: - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/install-valkey/action.yml - .github/workflows/lint-rust/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** - deny.toml pull_request: paths: @@ -24,39 +25,61 @@ on: - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/install-valkey/action.yml - .github/workflows/lint-rust/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** - deny.toml workflow_dispatch: + inputs: + full-matrix: + description: "Run the full engine and host matrix" + type: boolean + default: false + name: + required: false + type: string + description: "(Optional) Test run name" + + workflow_call: concurrency: - group: rust-${{ github.head_ref || github.ref }} + group: rust-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} cancel-in-progress: true env: CARGO_TERM_COLOR: always +run-name: + # Set custom name if job is started manually and name is given + ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }} + jobs: - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') || github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + # language version matrix is omitted - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: rust + # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow + run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} - build: - runs-on: ubuntu-latest - needs: load-engine-matrix + tests: + runs-on: ${{ matrix.host.RUNNER }} + needs: get-matrices timeout-minutes: 15 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} steps: - uses: actions/checkout@v4