diff --git a/.baedeker/collator-selection.jsonnet b/.baedeker/collator-selection.jsonnet index 2727b16947..3ee31041e4 100644 --- a/.baedeker/collator-selection.jsonnet +++ b/.baedeker/collator-selection.jsonnet @@ -12,27 +12,28 @@ local relay = { chain: relay_spec, modify:: bdk.mixer([ m.genericRelay($), - { - genesis+: { - runtimeGenesis+: { - runtime+: { - configuration+: { - config+: { - async_backing_params+: { + m.simplifyGenesisName(), + { + _genesis+: { + configuration+: { + config+: { + async_backing_params+: { allowed_ancestry_len: 3, max_candidate_depth: 4, }, - scheduling_lookahead:5, - max_validators_per_core:1, - minimum_backing_votes:1, - needed_approvals:1, - on_demand_cores:5, + validation_upgrade_cooldown: 200, + validation_upgrade_delay: 100, + minimum_validation_upgrade_delay: 15, + minimum_backing_votes: 2, + needed_approvals: 2, + scheduler_params+: { + lookahead: 1, + }, }, }, - }, - }, }, }, + m.unsimplifyGenesisName(), ]), }}, nodes: { diff --git a/.baedeker/external-runtime.jsonnet b/.baedeker/external-runtime.jsonnet index 3cb0692d5f..823ed45613 100644 --- a/.baedeker/external-runtime.jsonnet +++ b/.baedeker/external-runtime.jsonnet @@ -12,27 +12,28 @@ local relay = { chain: relay_spec, modify:: bdk.mixer([ m.genericRelay($), - { - genesis+: { - runtimeGenesis+: { - runtime+: { - configuration+: { - config+: { - async_backing_params+: { + m.simplifyGenesisName(), + { + _genesis+: { + configuration+: { + config+: { + async_backing_params+: { allowed_ancestry_len: 3, max_candidate_depth: 4, }, - scheduling_lookahead:5, - max_validators_per_core:1, - minimum_backing_votes:1, - needed_approvals:1, - on_demand_cores:5, + validation_upgrade_cooldown: 200, + validation_upgrade_delay: 100, + minimum_validation_upgrade_delay: 15, + minimum_backing_votes: 2, + needed_approvals: 2, + scheduler_params+: { + lookahead: 1, + }, }, }, - }, - }, }, }, + m.unsimplifyGenesisName(), ]), }}, nodes: { diff --git a/.baedeker/forkless-data.jsonnet b/.baedeker/forkless-data.jsonnet index 151ae92cb6..b0d0c093a5 100644 --- a/.baedeker/forkless-data.jsonnet +++ b/.baedeker/forkless-data.jsonnet @@ -11,7 +11,31 @@ local relay = { validatorIdAssignment: 'staking', spec: {Genesis:{ chain: relay_spec, - modify:: m.genericRelay($), + modify:: bdk.mixer([ + m.genericRelay($), + m.simplifyGenesisName(), + { + _genesis+: { + configuration+: { + config+: { + async_backing_params+: { + allowed_ancestry_len: 3, + max_candidate_depth: 4, + }, + validation_upgrade_cooldown: 200, + validation_upgrade_delay: 100, + minimum_validation_upgrade_delay: 15, + minimum_backing_votes: 2, + needed_approvals: 2, + scheduler_params+: { + lookahead: 1, + }, + }, + }, + }, + }, + m.unsimplifyGenesisName(), + ]), }}, nodes: { [name]: { diff --git a/.baedeker/just-tests.jsonnet b/.baedeker/just-tests.jsonnet index a151540dd3..7931c22042 100644 --- a/.baedeker/just-tests.jsonnet +++ b/.baedeker/just-tests.jsonnet @@ -13,32 +13,30 @@ local relay = { chain: relay_spec, modify:: bdk.mixer([ m.genericRelay($), - { - genesis+: { - runtimeGenesis+: { - runtime+: { - configuration+: { - config+: { - async_backing_params+: { + m.simplifyGenesisName(), + { + _genesis+: { + configuration+: { + config+: { + async_backing_params+: { allowed_ancestry_len: 3, max_candidate_depth: 4, }, - scheduling_lookahead:5, - max_validators_per_core:1, - minimum_backing_votes:1, - needed_approvals:1, - on_demand_cores:5, - validation_upgrade_cooldown:400, - validation_upgrade_delay:200, - minimum_validation_upgrade_delay:10, + validation_upgrade_cooldown: 200, + validation_upgrade_delay: 100, + minimum_validation_upgrade_delay: 15, + minimum_backing_votes: 2, + needed_approvals: 2, + scheduler_params+: { + lookahead: 1, + }, }, }, - }, - }, }, }, + m.unsimplifyGenesisName(), ]), - }}, + }}, nodes: { [name]: { bin: $.bin, @@ -94,7 +92,6 @@ local unique = { extraArgs: [ '--increase-future-pool', '--pool-type=fork-aware', - '-laura=debug', ], }, for name in ['alice', 'bob', 'charlie'] diff --git a/.baedeker/node-only.jsonnet b/.baedeker/node-only.jsonnet index 7e364ea380..c29159c012 100644 --- a/.baedeker/node-only.jsonnet +++ b/.baedeker/node-only.jsonnet @@ -10,7 +10,31 @@ local relay = { validatorIdAssignment: 'staking', spec: {Genesis:{ chain: relay_spec, - modify:: m.genericRelay($), + modify:: bdk.mixer([ + m.genericRelay($), + m.simplifyGenesisName(), + { + _genesis+: { + configuration+: { + config+: { + async_backing_params+: { + allowed_ancestry_len: 3, + max_candidate_depth: 4, + }, + validation_upgrade_cooldown: 200, + validation_upgrade_delay: 100, + minimum_validation_upgrade_delay: 15, + minimum_backing_votes: 2, + needed_approvals: 2, + scheduler_params+: { + lookahead: 1, + }, + }, + }, + }, + }, + m.unsimplifyGenesisName(), + ]), }}, nodes: { [name]: { diff --git a/.baedeker/xcm-opal.jsonnet b/.baedeker/xcm-opal.jsonnet index 1c7c724b54..73f328cf09 100644 --- a/.baedeker/xcm-opal.jsonnet +++ b/.baedeker/xcm-opal.jsonnet @@ -38,6 +38,7 @@ local unique = { wantedKeys: 'para', extraArgs: [ '--increase-future-pool', + '--pool-type=fork-aware', ], }, for name in ['alice', 'bob'] diff --git a/.baedeker/xcm-quartz.jsonnet b/.baedeker/xcm-quartz.jsonnet index fa9982a8a8..2b260373c0 100644 --- a/.baedeker/xcm-quartz.jsonnet +++ b/.baedeker/xcm-quartz.jsonnet @@ -26,7 +26,7 @@ local relay = { wantedKeys: 'relay', expectedDataPath: '/parity', }, - for name in ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie', 'gregory', 'holly', 'iggy', 'john', 'kurt'] + for name in ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie'] }, }; @@ -43,6 +43,7 @@ local unique = { wantedKeys: 'para', extraArgs: [ '--increase-future-pool', + '--pool-type=fork-aware', ], }, for name in ['alice', 'bob'] diff --git a/.baedeker/xcm-unique.jsonnet b/.baedeker/xcm-unique.jsonnet index 04356ddba8..75e75d7cb0 100644 --- a/.baedeker/xcm-unique.jsonnet +++ b/.baedeker/xcm-unique.jsonnet @@ -27,7 +27,7 @@ local relay = { wantedKeys: 'relay', expectedDataPath: '/parity', }, - for name in ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie', 'gregory', 'holly', 'iggy', 'john', 'kurt', 'larry', 'mike', 'norman', 'osvald'] + for name in ['alice', 'bob', 'charlie', 'dave', 'eve', 'ferdie'] }, }; @@ -44,6 +44,7 @@ local unique = { wantedKeys: 'para', extraArgs: [ '--increase-future-pool', + '--pool-type=fork-aware', ], }, for name in ['alice', 'bob'] diff --git a/.github/workflows/ci-develop.yml b/.github/workflows/ci-develop.yml index d8f605a1d0..6ebc01582c 100644 --- a/.github/workflows/ci-develop.yml +++ b/.github/workflows/ci-develop.yml @@ -41,35 +41,11 @@ jobs: uses: ./.github/workflows/collator-selection.yml secrets: inherit -#### should be removed #### - - forkless-update-data: - if: github.event.pull_request.draft == false && (contains(github.event.pull_request.labels.*.name, 'CI-forkless-data') || contains(github.event.pull_request.labels.*.name, 'CI-forkless-update-data')) - uses: ./.github/workflows/forkless-update-data.yml - secrets: inherit - - forkless-update-no-data: - if: github.event.pull_request.draft == false && (contains(github.event.pull_request.labels.*.name, 'CI-forkless-no-data') || contains(github.event.pull_request.labels.*.name, 'CI-forkless-update-no-data')) - uses: ./.github/workflows/forkless-update-nodata.yml - secrets: inherit - - node-only-update: - if: github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'CI-node-only-update') - uses: ./.github/workflows/node-only-update.yml - secrets: inherit - -#### end #### - main-process: if: github.event.pull_request.draft == false && (contains(github.event.pull_request.labels.*.name, 'CI-main-process') || contains(github.event.pull_request.labels.*.name, 'CI-main-process')) uses: ./.github/workflows/main-process-update.yml secrets: inherit - just-tests: - if: github.event.pull_request.draft == false && (contains(github.event.pull_request.labels.*.name, 'CI-just-tests') || contains(github.event.pull_request.labels.*.name, 'CI-just-tests')) - uses: ./.github/workflows/just-tests-without-update.yml - secrets: inherit - try-runtime: if: github.event.pull_request.draft == false && (contains(github.event.pull_request.labels.*.name, 'CI-try-runtime') || contains(github.event.pull_request.labels.*.name, 'CI-try-runtime')) uses: ./.github/workflows/try-runtime.yml diff --git a/.github/workflows/ci-master.yml b/.github/workflows/ci-master.yml index acf5382149..5dc243062f 100644 --- a/.github/workflows/ci-master.yml +++ b/.github/workflows/ci-master.yml @@ -36,22 +36,14 @@ jobs: uses: ./.github/workflows/collator-selection.yml secrets: inherit - forkless-update-data: - uses: ./.github/workflows/forkless-update-data.yml - secrets: inherit # pass all secrets from initial workflow to nested - - forkless-update-no-data: - uses: ./.github/workflows/forkless-update-nodata.yml - secrets: inherit # pass all secrets from initial workflow to nested + main-process: + uses: ./.github/workflows/main-process-update.yml + secrets: inherit try-runtime: uses: ./.github/workflows/try-runtime.yml secrets: inherit # pass all secrets from initial workflow to nested - node-only-update: - uses: ./.github/workflows/node-only-update.yml - secrets: inherit - governance: uses: ./.github/workflows/governance.yml secrets: inherit diff --git a/.github/workflows/forkless-update-data.yml b/.github/workflows/forkless-update-data.yml deleted file mode 100644 index ce62ae6b0f..0000000000 --- a/.github/workflows/forkless-update-data.yml +++ /dev/null @@ -1,226 +0,0 @@ -# Forkless update with data replication -# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586869792/Forkless+update+with+data - -# Triger: only call from main workflow(re-usable workflows) -name: forkless-update-data - -on: - workflow_call: - workflow_dispatch: - - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - - prepare-execution-matrix: - - name: execution matrix - - runs-on: [ ci ] - outputs: - matrix: ${{ steps.create_matrix.outputs.matrix }} - - steps: - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Create Execution matrix - uses: CertainLach/create-matrix-action@v4 - id: create_matrix - with: - matrix: | - network {opal}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}, fork_source {${{ env.OPAL_REPLICA_FROM }}} - network {quartz}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}, fork_source {${{ env.QUARTZ_REPLICA_FROM }}} - network {unique}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}, fork_source {${{ env.UNIQUE_REPLICA_FROM }}} - - forkless-data: - - needs: prepare-execution-matrix - # The type of runner that the job will run on - runs-on: [ ci ] - - timeout-minutes: 1380 - - name: ${{ matrix.network }}-data - - continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. - - strategy: - matrix: - include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}} - - steps: - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - # Prepare SHA - - name: Prepare SHA - uses: ./.github/actions/prepare - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Log in to Docker Hub - uses: docker/login-action@v3.2.0 - with: - username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Check and pull polkadot image - id: polkadot - uses: cloudposse/github-action-docker-image-exists@main - with: - registry: registry.hub.docker.com - organization: parity - repository: polkadot - login: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - tag: ${{ matrix.relay_branch }} - - - name: Prepare mainnet - uses: ./.github/actions/buildContainer - id: mainnet - with: - container: uniquenetwork/ci-forkless-data - tag: ${{ matrix.network }}-${{ env.MAINNET_BRANCH }}-${{ env.MAINNET_HASH}} - context: .docker - dockerfile: Dockerfile-unique-release - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - --build-arg CODE_VERSION=${{ env.MAINNET_BRANCH}} - --build-arg CODE_HASH=${{ env.MAINNET_HASH}} - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Prepare target runtime - uses: ./.github/actions/buildContainer - id: runtime - with: - container: uniquenetwork/ci-forkless-data - tag: ${{ matrix.network }}-${{ env.REF_SLUG }}-${{ env.BUILD_SHA }} - context: . - dockerfile: .docker/Dockerfile-unique - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Extract wasms - uses: ./.github/actions/extractDocker - id: wasms - with: - image: ${{ steps.runtime.outputs.name }} - directory: /wasm - - - name: Prepare target node - uses: ./.github/actions/buildContainer - id: node - with: - container: uniquenetwork/ci-forkless-data - tag: ${{ matrix.network }}-${{ env.NODE_BRANCH }}-${{ env.NODE_HASH}} - context: .docker - dockerfile: Dockerfile-unique-release - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - --build-arg CODE_VERSION=${{ env.NODE_BRANCH}} - --build-arg CODE_HASH=${{ env.NODE_HASH}} - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - uses: actions/setup-node@v4.0.2 - with: - node-version: 20 - - - name: Install baedeker - uses: UniqueNetwork/baedeker-action/setup@v1.2 - with: - version: "v0.1.4" - useCache: false - - - name: Setup library - run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library - - - name: Start network - uses: UniqueNetwork/baedeker-action@v1.2 - if: success() - id: bdk - with: - jpath: | - .baedeker/vendor - tla-str: | - relay_spec=${{ env.RELAY_CHAIN_TYPE }}-local - forked_spec=${{ matrix.network }} - fork_source=${{ matrix.fork_source }} - inputs: | - .baedeker/forkless-data.jsonnet - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.image }}:${{ steps.polkadot.outputs.tag }}'}}) - # with data build uses old runtime, but new node, thus we use mainnet image for spec generation, and then target image for nodes. - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.node.outputs.name }}'}}, extra_node_mixin={extraArgs: []}, for_chain = false) - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, extra_node_mixin={extraArgs: []}) - - - name: Ensure network is alive - working-directory: js-packages/tests - id: alive1 - if: ${{ !cancelled() && steps.bdk.outcome == 'success' }} - run: | - yarn - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: "Reconcile: runtime is upgraded" - working-directory: js-packages/tests - id: reconcile1 - if: ${{ !cancelled() && steps.alive1.outcome == 'success' }} - run: | - echo "Executing upgrade" - yarn node --no-warnings=ExperimentalWarning --loader ts-node/esm ../scripts/authorizeEnactUpgrade.ts ${{ steps.wasms.outputs.dir }}/${{ matrix.network }}-runtime/${{ matrix.network }}_runtime.compact.compressed.wasm - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Ensure network is alive - working-directory: js-packages/tests - id: alive2 - if: ${{ !cancelled() && steps.reconcile1.outcome == 'success' }} - run: | - yarn - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Parallel tests after forkless upgrade - working-directory: js-packages/tests - if: ${{ !cancelled() && steps.alive2.outcome == 'success' }} - run: | - echo "Ready to start tests" - NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Sequential tests after forkless upgrade - working-directory: js-packages/tests - if: ${{ !cancelled() && steps.alive2.outcome == 'success' }} - run: | - NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Remove builder cache - if: always() # run this step always - run: | - docker system prune -f diff --git a/.github/workflows/forkless-update-nodata.yml b/.github/workflows/forkless-update-nodata.yml deleted file mode 100644 index 7ba711d887..0000000000 --- a/.github/workflows/forkless-update-nodata.yml +++ /dev/null @@ -1,224 +0,0 @@ -# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586837021/Forkless+update+without+data -# Forkless update without data replication - -# Triger: only call from main workflow(re-usable workflows) -name: forkless-update-nodata - -on: - workflow_call: - workflow_dispatch: - - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - - prepare-execution-matrix: - - name: execution matrix - - runs-on: [ ci ] - outputs: - matrix: ${{ steps.create_matrix.outputs.matrix }} - - steps: - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Create Execution matrix - uses: CertainLach/create-matrix-action@v4 - id: create_matrix - with: - matrix: | - network {opal}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}} - network {quartz}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}} - network {unique}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}} - - forkless-nodata: - - needs: prepare-execution-matrix - # The type of runner that the job will run on - runs-on: [ ci ] - - timeout-minutes: 1380 - - name: ${{ matrix.network }}-nodata - - continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. - - strategy: - matrix: - include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}} - - steps: - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - # Prepare SHA - - name: Prepare SHA - uses: ./.github/actions/prepare - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Log in to Docker Hub - uses: docker/login-action@v3.2.0 - with: - username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Check and pull polkadot image - id: polkadot - uses: cloudposse/github-action-docker-image-exists@main - with: - registry: registry.hub.docker.com - organization: parity - repository: polkadot - login: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - tag: ${{ matrix.relay_branch }} - - - name: Prepare mainnet - uses: ./.github/actions/buildContainer - id: mainnet - with: - container: uniquenetwork/ci-forkless-nodata - tag: ${{ matrix.network }}-${{ env.MAINNET_BRANCH }}-${{ env.MAINNET_HASH}} - context: .docker - dockerfile: Dockerfile-unique-release - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - --build-arg CODE_VERSION=${{ env.MAINNET_BRANCH}} - --build-arg CODE_HASH=${{ env.MAINNET_HASH}} - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Prepare target runtime - uses: ./.github/actions/buildContainer - id: runtime - with: - container: uniquenetwork/ci-forkless-nodata - tag: ${{ matrix.network }}-${{ env.REF_SLUG }}-${{ env.BUILD_SHA }} - context: . - dockerfile: .docker/Dockerfile-unique - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Extract wasms - uses: ./.github/actions/extractDocker - id: wasms - with: - image: ${{ steps.runtime.outputs.name }} - directory: /wasm - - - name: Prepare target node - uses: ./.github/actions/buildContainer - id: node - with: - container: uniquenetwork/ci-forkless-nodata - tag: ${{ matrix.network }}-${{ env.NODE_BRANCH }}-${{ env.NODE_HASH}} - context: .docker - dockerfile: Dockerfile-unique-release - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - --build-arg CODE_VERSION=${{ env.NODE_BRANCH}} - --build-arg CODE_HASH=${{ env.NODE_HASH}} - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - uses: actions/setup-node@v4.0.2 - with: - node-version: 20 - - - name: Install baedeker - uses: UniqueNetwork/baedeker-action/setup@v1.2 - with: - version: "v0.1.4" - useCache: false - - - name: Setup library - run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library - - - name: Start network - uses: UniqueNetwork/baedeker-action@v1.2 - id: bdk - if: success() - with: - jpath: | - .baedeker/vendor - tla-str: | - relay_spec=${{ env.RELAY_CHAIN_TYPE }}-local - inputs: | - .baedeker/node-only.jsonnet - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.image }}:${{ steps.polkadot.outputs.tag }}'}}) - # nodata build uses old runtime, but new chain, thus we use mainnet image for spec generation, and then target image for nodes. - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.node.outputs.name }}'}}, extra_node_mixin={extraArgs: []}, for_chain = false) - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, extra_node_mixin={extraArgs: []}) - - - name: Ensure network is alive - working-directory: js-packages/tests - id: alive1 - if: ${{ !cancelled() && steps.bdk.outcome == 'success' }} - run: | - yarn - yarn add mochawesome - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: "Reconcile: runtime is upgraded" - working-directory: js-packages/tests - id: reconcile1 - if: ${{ !cancelled() && steps.alive1.outcome == 'success' }} - run: | - echo "Executing upgrade" - yarn node --no-warnings=ExperimentalWarning --loader ts-node/esm ../scripts/authorizeEnactUpgrade.ts ${{ steps.wasms.outputs.dir }}/${{ matrix.network }}-runtime/${{ matrix.network }}_runtime.compact.compressed.wasm - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Ensure network is alive - working-directory: js-packages/tests - id: alive2 - if: ${{ !cancelled() && steps.reconcile1.outcome == 'success' }} - run: | - yarn - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Parallel tests after forkless upgrade - working-directory: js-packages/tests - if: ${{ !cancelled() && steps.alive2.outcome == 'success' }} - run: | - echo "Ready to start tests" - NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Sequential tests after forkless upgrade - working-directory: js-packages/tests - if: ${{ !cancelled() && steps.alive2.outcome == 'success' }} - run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Remove builder cache - if: always() # run this step always - run: | - docker system prune -f diff --git a/.github/workflows/just-tests-without-update.yml b/.github/workflows/just-tests-without-update.yml deleted file mode 100644 index c9c2487524..0000000000 --- a/.github/workflows/just-tests-without-update.yml +++ /dev/null @@ -1,225 +0,0 @@ -# Triger: only call from main workflow(re-usable workflows) -name: main-process-update - -on: - workflow_call: - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - - prepare-execution-matrix: - - name: execution matrix - - runs-on: [ ci ] - outputs: - matrix: ${{ steps.create_matrix.outputs.matrix }} - - steps: - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Create Execution matrix - uses: CertainLach/create-matrix-action@v4 - id: create_matrix - with: - matrix: | - network {opal}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}, fork_source {${{ env.OPAL_REPLICA_FROM }}} - network {quartz}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}, fork_source {${{ env.QUARTZ_REPLICA_FROM }}} - network {unique}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}, fork_source {${{ env.UNIQUE_REPLICA_FROM }}} - - main-process: - - needs: prepare-execution-matrix - # The type of runner that the job will run on - runs-on: [ ci ] - - timeout-minutes: 1380 - - name: ${{ matrix.network }}-tests - - continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. - - strategy: - matrix: - include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}} - - steps: - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - # Prepare SHA - - name: Prepare SHA - uses: ./.github/actions/prepare - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Log in to Docker Hub - uses: docker/login-action@v3.2.0 - with: - username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Check and pull polkadot image - id: polkadot - uses: cloudposse/github-action-docker-image-exists@main - with: - registry: registry.hub.docker.com - organization: parity - repository: polkadot - login: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - tag: ${{ matrix.relay_branch }} - - - name: Prepare target runtime - uses: ./.github/actions/buildContainer - id: runtime - with: - container: uniquenetwork/ci-main-process - tag: ${{ matrix.network }}-${{ env.REF_SLUG }}-${{ env.BUILD_SHA }} - context: . - dockerfile: .docker/Dockerfile-unique - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Extract wasms - uses: ./.github/actions/extractDocker - id: wasms - with: - image: ${{ steps.runtime.outputs.name }} - directory: /wasm - - - name: Prepare target node - uses: ./.github/actions/buildContainer - id: node - with: - container: uniquenetwork/ci-main-process - tag: ${{ matrix.network }}-${{ env.NODE_BRANCH }}-${{ env.NODE_HASH}} - context: .docker - dockerfile: Dockerfile-unique-release - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - --build-arg CODE_VERSION=${{ env.NODE_BRANCH}} - --build-arg CODE_HASH=${{ env.NODE_HASH}} - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - uses: actions/setup-node@v4.0.2 - with: - node-version: 20 - - - name: Install baedeker - uses: UniqueNetwork/baedeker-action/setup@v1.2 - with: - version: "v0.1.4" - useCache: false - - - name: Setup library - run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library - - - name: Get state from production - run: | - . "$HOME/.cargo/env" - chainql -e "cql.chain(\"${{ matrix.fork_source }}\").latest._preloadKeys._raw" > .baedeker/vendor/dump.json - - - name: Copy runtime file - run: | - cp ${{ steps.wasms.outputs.dir }}/${{ matrix.network }}-runtime/${{ matrix.network }}_runtime.compact.compressed.wasm .baedeker/vendor/runtime.compact.compressed.wasm - - - name: Start network - uses: UniqueNetwork/baedeker-action@v1.2 - if: success() - id: bdk - with: - jpath: | - .baedeker/vendor - input-modules: | - lib:baedeker-library/ops/nginx-dev.libsonnet - tla-str: | - relay_spec=${{ env.RELAY_CHAIN_TYPE }}-local - forked_spec=${{ matrix.network }} - tla-code: | - dump_spec=import 'dump.json' - inputs: | - .baedeker/just-tests.jsonnet - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.image }}:${{ steps.polkadot.outputs.tag }}'}}) - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.node.outputs.name }}'}}, extra_node_mixin={extraArgs: []}) - - - name: Ensure network is alive - working-directory: js-packages/tests - id: alive3 - if: ${{ !cancelled() && steps.reconcile2.outcome == 'success' }} - run: | - yarn - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Parallel tests after upgrade - working-directory: js-packages/tests - if: ${{ !cancelled() && steps.alive3.outcome == 'success' }} - run: | - echo "adding mochawesome" - yarn add mochawesome - echo "Ready to start Parallel tests" - NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Parallel Test Report - uses: phoenix-actions/test-reporting@v15 - id: parallel-report - if: success() || failure() - with: - name: Main Parallel Tests ${{ matrix.network }} - path: js-packages/tests/mochawesome-report/test-parallel-*.json - reporter: mochawesome-json - fail-on-error: 'false' - - - name: Run Sequential tests after upgrade - working-directory: js-packages/tests - if: ${{ !cancelled() && steps.alive3.outcome == 'success' }} - run: | - echo "Ready to start Sequential tests" - NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Sequential Tests Report - uses: phoenix-actions/test-reporting@v15 - id: sequential-report - if: success() || failure() - with: - name: Main Sequential Tests ${{ matrix.network }} - path: js-packages/tests/mochawesome-report/test-sequential-*.json - reporter: mochawesome-json - fail-on-error: 'false' - - - name: Remove all containers - if: always() # run this step always - run: | - docker ps -aq | xargs docker rm -f - - - name: Remove builder cache - if: always() # run this step always - run: | - docker system prune -f -a diff --git a/.github/workflows/node-only-update.yml b/.github/workflows/node-only-update.yml deleted file mode 100644 index e9a5e00c1a..0000000000 --- a/.github/workflows/node-only-update.yml +++ /dev/null @@ -1,250 +0,0 @@ -# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586837028/Nodes+only+update -# Node only update with restart polkadot-launch process. - -name: node-only-update - -# Triger: only call from main workflow(re-usable workflows) -on: - workflow_call: - workflow_dispatch: - - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - - prepare-execution-matrix: - - name: execution matrix - - runs-on: [ ci ] - outputs: - matrix: ${{ steps.create_matrix.outputs.matrix }} - - steps: - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Create Execution matrix - uses: CertainLach/create-matrix-action@v4 - id: create_matrix - with: - matrix: | - network {opal}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}} - network {quartz}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}} - network {unique}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}} - - node-only-update: - - needs: prepare-execution-matrix - # The type of runner that the job will run on - runs-on: [ ci ] - - timeout-minutes: 2880 # 48 hours for execution jobs. - - name: ${{ matrix.network }} - - continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. - - strategy: - matrix: - include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}} - - steps: - - - name: Skip if pull request is in Draft - if: github.event.pull_request.draft == true - run: exit 1 - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - # Prepare SHA - - name: Prepare SHA - uses: ./.github/actions/prepare - - - name: Read .env file - uses: xom9ikk/dotenv@v2.3.0 - - - name: Log in to Docker Hub - uses: docker/login-action@v3.2.0 - with: - username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Check and pull polkadot image - id: polkadot - uses: cloudposse/github-action-docker-image-exists@main - with: - registry: registry.hub.docker.com - organization: parity - repository: polkadot - login: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - tag: ${{ matrix.relay_branch }} - - - name: Prepare mainnet node - uses: ./.github/actions/buildContainer - id: mainnet - with: - container: uniquenetwork/ci-node-only - tag: ${{ matrix.network }}-${{ env.MAINNET_BRANCH }}-${{ env.MAINNET_HASH}} - context: .docker - dockerfile: Dockerfile-unique-release - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - --build-arg CODE_VERSION=${{ env.MAINNET_BRANCH}} - --build-arg CODE_HASH=${{ env.MAINNET_HASH}} - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Prepare target node - uses: ./.github/actions/buildContainer - id: target - with: - container: uniquenetwork/ci-node-only - tag: ${{ matrix.network }}-${{ env.NODE_BRANCH }}-${{ env.NODE_HASH}} - context: .docker - dockerfile: Dockerfile-unique-release - args: | - --build-arg FEATURES=${{ matrix.network }}-runtime - --build-arg CODE_VERSION=${{ env.NODE_BRANCH}} - --build-arg CODE_HASH=${{ env.NODE_HASH}} - dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Checkout at '${{ env.MAINNET_BRANCH }}' branch - uses: actions/checkout@v4.1.7 - with: - #ref: ${{ github.head_ref }} - ref: ${{ env.MAINNET_BRANCH }} #Checking out head commit - path: ${{ env.MAINNET_BRANCH }} - - - uses: actions/setup-node@v4.0.2 - with: - node-version: 20 - - - name: Install baedeker - uses: UniqueNetwork/baedeker-action/setup@v1.2 - with: - version: "v0.1.4" - useCache: false - - - name: Setup library - run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library - - - name: Start network - uses: UniqueNetwork/baedeker-action@v1.2 - if: success() - id: bdk - with: - jpath: | - .baedeker/vendor - tla-str: | - relay_spec=${{ env.RELAY_CHAIN_TYPE }}-local - inputs: | - .baedeker/node-only.jsonnet - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.image }}:${{ steps.polkadot.outputs.tag }}'}}) - ephemeral:snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, extra_node_mixin={extraArgs: []}) - - - name: Ensure network is alive - working-directory: ${{ env.MAINNET_BRANCH }}/js-packages/tests - id: alive1 - if: ${{ !cancelled() && steps.bdk.outcome == 'success' }} - run: | - yarn - yarn add mochawesome - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Parallel tests before Node Parachain upgrade - working-directory: ${{ env.MAINNET_BRANCH }}/js-packages/tests - if: ${{ !cancelled() && steps.alive1.outcome == 'success' }} - run: | - echo "Ready to start tests" - NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Sequential tests before Node Parachain upgrade - if: ${{ !cancelled() && steps.alive1.outcome == 'success' }} - working-directory: ${{ env.MAINNET_BRANCH }}/js-packages/tests - run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: "Reconcile: only one old node" - if: ${{ !cancelled() && steps.alive1.outcome == 'success' }} - id: reconcile1 - uses: UniqueNetwork/baedeker-action/reconcile@v1.2 - with: - baedeker: ${{ steps.bdk.outputs.baedeker }} - # Chain should always be built with the mainnet spec, this we first set binary for all nodes expect one, then set mainnet binary for the last node, and then force chainspec to be still generated from mainnet - inputs: | - ephemeral:snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.target.outputs.name }}'}}, leave = 1, for_chain = false) - ephemeral:snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, for_chain = false) - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}) - - - name: Ensure network is alive - working-directory: ${{ env.MAINNET_BRANCH }}/js-packages/tests - id: alive2 - if: ${{ !cancelled() && steps.reconcile1.outcome == 'success' }} - run: | - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: "Reconcile: all nodes are updated" - if: ${{ !cancelled() && steps.alive2.outcome == 'success' }} - id: reconcile2 - uses: UniqueNetwork/baedeker-action/reconcile@v1.2 - with: - baedeker: ${{ steps.bdk.outputs.baedeker }} - # Chain should always be built with the mainnet spec, this we first set binary for all nodes, and then force chainspec to be still generated from mainnet - inputs: | - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.target.outputs.name }}'}}, for_chain = false) - snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}) - - - name: Ensure network is alive - working-directory: ${{ env.MAINNET_BRANCH }}/js-packages/tests - id: alive3 - if: ${{ !cancelled() && steps.reconcile2.outcome == 'success' }} - run: | - ../scripts/wait_for_first_block.sh - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Parallel tests after Node Parachain upgrade - working-directory: ${{ env.MAINNET_BRANCH }}/js-packages/tests - if: ${{ !cancelled() && steps.alive3.outcome == 'success' }} - run: | - echo "Ready to start tests" - NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Run Sequential tests after Node Parachain upgrade - if: ${{ !cancelled() && steps.alive3.outcome == 'success' }} - working-directory: ${{ env.MAINNET_BRANCH }}/js-packages/tests - run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW} - env: - RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - - name: Remove builder cache - if: always() # run this step always - run: | - docker system prune -f -a