From 01dd59cd5fdc6dbca5d5b614faaf45b291b67768 Mon Sep 17 00:00:00 2001 From: zeroxbt Date: Mon, 17 Jul 2023 22:16:02 +0200 Subject: [PATCH 1/5] only start libp2p after router init --- ot-node.js | 6 ++++++ src/modules/network/implementation/libp2p-service.js | 5 ++++- src/modules/network/network-module-manager.js | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ot-node.js b/ot-node.js index 2660f544ad..5e89a6b99f 100644 --- a/ot-node.js +++ b/ot-node.js @@ -69,6 +69,7 @@ class OTNode { await this.initializeCommandExecutor(); await this.initializeRouters(); + await this.startNetworkModule(); this.logger.info('Node is up and running!'); } @@ -254,6 +255,11 @@ class OTNode { } } + async startNetworkModule() { + const networkModuleManager = this.container.resolve('networkModuleManager'); + await networkModuleManager.start(); + } + async executePrivateAssetsMetadataMigration() { if ( process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || diff --git a/src/modules/network/implementation/libp2p-service.js b/src/modules/network/implementation/libp2p-service.js index 1b99a1c873..1d3885debe 100644 --- a/src/modules/network/implementation/libp2p-service.js +++ b/src/modules/network/implementation/libp2p-service.js @@ -101,13 +101,16 @@ class Libp2pService { */ this.sessions = {}; this.node = await libp2p.create(initializationObject); - await this.node.start(); const port = parseInt(this.node.multiaddrs.toString().split('/')[4], 10); const peerId = this.node.peerId.toB58String(); this.config.id = peerId; this.logger.info(`Network ID is ${peerId}, connection port is ${port}`); } + async start() { + this.node.start(); + } + async onPeerConnected(listener) { this.node.connectionManager.on('peer:connect', listener); } diff --git a/src/modules/network/network-module-manager.js b/src/modules/network/network-module-manager.js index 758ca4244a..698c7f5697 100644 --- a/src/modules/network/network-module-manager.js +++ b/src/modules/network/network-module-manager.js @@ -5,6 +5,12 @@ class NetworkModuleManager extends BaseModuleManager { return 'network'; } + async start() { + if (this.initialized) { + return this.getImplementation().module.start(); + } + } + async onPeerConnected(listener) { if (this.initialized) { return this.getImplementation().module.onPeerConnected(listener); From 82fa8e84af4799b4c4bef7089280aa15f1e67595 Mon Sep 17 00:00:00 2001 From: zeroxbt Date: Mon, 17 Jul 2023 22:26:23 +0200 Subject: [PATCH 2/5] fix update peer last seen and dialed --- .../repositories/shard-repository.js | 4 ++-- src/service/sharding-table-service.js | 21 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index d2a00d74e2..08dc04df82 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -108,7 +108,7 @@ class ShardRepository { } async updatePeerRecordLastDialed(peerId, timestamp) { - await this.model.update( + return this.model.update( { lastDialed: timestamp, }, @@ -119,7 +119,7 @@ class ShardRepository { } async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp) { - await this.model.update( + return this.model.update( { lastDialed: timestamp, lastSeen: timestamp, diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 2a09e598ef..00bab83bcc 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -214,15 +214,21 @@ class ShardingTableService { }; } if (this.memoryCachedPeerIds[peerId].lastUpdated < timestampThreshold) { - await this.repositoryModuleManager.updatePeerRecordLastSeenAndLastDialed(peerId, now); - this.memoryCachedPeerIds[peerId].lastUpdated = now; + const [rowsUpdated] = + await this.repositoryModuleManager.updatePeerRecordLastSeenAndLastDialed( + peerId, + now, + ); + if (rowsUpdated) { + this.memoryCachedPeerIds[peerId].lastUpdated = now; + } } this.memoryCachedPeerIds[peerId].lastDialed = now; this.memoryCachedPeerIds[peerId].lastSeen = now; } async updatePeerRecordLastDialed(peerId) { - const now = new Date(); + const now = Date.now(); const timestampThreshold = now - PEER_RECORD_UPDATE_DELAY; if (!this.memoryCachedPeerIds[peerId]) { this.memoryCachedPeerIds[peerId] = { @@ -232,8 +238,13 @@ class ShardingTableService { }; } if (this.memoryCachedPeerIds[peerId].lastUpdated < timestampThreshold) { - await this.repositoryModuleManager.updatePeerRecordLastDialed(peerId, now); - this.memoryCachedPeerIds[peerId].lastUpdated = now; + const [rowsUpdated] = await this.repositoryModuleManager.updatePeerRecordLastDialed( + peerId, + now, + ); + if (rowsUpdated) { + this.memoryCachedPeerIds[peerId].lastUpdated = now; + } } this.memoryCachedPeerIds[peerId].lastDialed = now; } From 8bddc8cda84117e779bf899097622c644015f4a3 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:34:41 +0200 Subject: [PATCH 3/5] fix network port log (#2666) * only start libp2p after router init * fix update peer last seen and dialed * fix network port log --- src/modules/network/implementation/libp2p-service.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/network/implementation/libp2p-service.js b/src/modules/network/implementation/libp2p-service.js index 1d3885debe..da50372d0a 100644 --- a/src/modules/network/implementation/libp2p-service.js +++ b/src/modules/network/implementation/libp2p-service.js @@ -101,14 +101,14 @@ class Libp2pService { */ this.sessions = {}; this.node = await libp2p.create(initializationObject); - const port = parseInt(this.node.multiaddrs.toString().split('/')[4], 10); const peerId = this.node.peerId.toB58String(); this.config.id = peerId; - this.logger.info(`Network ID is ${peerId}, connection port is ${port}`); } async start() { - this.node.start(); + await this.node.start(); + const port = parseInt(this.node.multiaddrs.toString().split('/')[4], 10); + this.logger.info(`Network ID is ${this.config.id}, connection port is ${port}`); } async onPeerConnected(listener) { From 1f1221d9d4c4dec7419029593e1b57c5c50c9aba Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Thu, 17 Aug 2023 12:04:42 +0200 Subject: [PATCH 4/5] Added updates for bid suggestion and assertion size handling (#2668) * Added updates for bid suggestion and assertion size handling * Added unit test for 1b 1 wei cases * Added logs to bid suggestion calculation * Updated wait time for bdd tests before network is ready * Added logs for bid suggestion calculation * Added filtering of peer id from find nodes in getbid suggestion * Unit tests fixed * Added fix for ask * Bug fixed * Logs removed --- .../common/handle-protocol-message-command.js | 3 +- src/service/sharding-table-service.js | 31 +++++++++----- test/bdd/features/get-errors.feature | 12 +++--- test/bdd/features/get.feature | 6 +-- test/bdd/features/publish-errors.feature | 6 +-- test/bdd/features/publish.feature | 2 +- test/bdd/features/update-errors.feature | 2 +- test/bdd/features/update.feature | 2 +- .../mock/blockchain-module-manager-mock.js | 4 ++ test/unit/mock/network-module-manager-mock.js | 8 +++- .../service/sharding-table-service.test.js | 40 ++++++++++++++++--- 11 files changed, 83 insertions(+), 33 deletions(-) diff --git a/src/commands/protocols/common/handle-protocol-message-command.js b/src/commands/protocols/common/handle-protocol-message-command.js index f6a559d2d8..79400fda6e 100644 --- a/src/commands/protocols/common/handle-protocol-message-command.js +++ b/src/commands/protocols/common/handle-protocol-message-command.js @@ -155,8 +155,7 @@ class HandleProtocolMessageCommand extends Command { .mul(epochsLeft) .mul(blockchainAssertionSize); - const serviceAgreementBid = this.blockchainModuleManager - .toBigNumber(blockchain, agreementData.tokenAmount) + const serviceAgreementBid = agreementData.tokenAmount .add(agreementData.updateTokenAmount) .mul(1024) .div(divisor) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 00bab83bcc..a1200c0deb 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -152,7 +152,6 @@ class ShardingTableService { firstAssertionId, hashFunctionId, ) { - const kbSize = assertionSize < BYTES_IN_KILOBYTE ? BYTES_IN_KILOBYTE : assertionSize; const peerRecords = await this.findNeighbourhood( blockchainId, this.blockchainModuleManager.encodePacked( @@ -164,20 +163,34 @@ class ShardingTableService { hashFunctionId, true, ); - - const sorted = peerRecords.sort((a, b) => a.ask - b.ask); - - const { ask } = sorted[Math.floor(sorted.length * 0.75)]; + const r1 = await this.blockchainModuleManager.getR1(blockchainId); + // todo remove this line once we implement logic for storing assertion in publish node if it's in neighbourhood + const myPeerId = this.networkModuleManager.getPeerId().toB58String(); + const filteredPeerRecords = peerRecords.filter((peer) => peer.peerId !== myPeerId); + const sorted = filteredPeerRecords.sort((a, b) => a.ask - b.ask); + let ask; + if (sorted.length > r1) { + ask = sorted[r1 - 1].ask; + } else { + ask = sorted[sorted.length - 1].ask; + } const r0 = await this.blockchainModuleManager.getR0(blockchainId); - return this.blockchainModuleManager + const minBidSuggestion = this.blockchainModuleManager + .toBigNumber(blockchainId, '1') + .mul(epochsNumber) + .mul(r0); + + const bidSuggestion = this.blockchainModuleManager .toBigNumber(blockchainId, this.blockchainModuleManager.convertToWei(blockchainId, ask)) - .mul(kbSize) + .mul(assertionSize) .mul(epochsNumber) .mul(r0) - .div(BYTES_IN_KILOBYTE) - .toString(); + .div(BYTES_IN_KILOBYTE); + return bidSuggestion.lte(minBidSuggestion) + ? minBidSuggestion.toString() + : bidSuggestion.toString(); } async findEligibleNodes(neighbourhood, bid, r1, r0) { diff --git a/test/bdd/features/get-errors.feature b/test/bdd/features/get-errors.feature index 27b8516da3..1ea6d7c22f 100644 --- a/test/bdd/features/get-errors.feature +++ b/test/bdd/features/get-errors.feature @@ -6,16 +6,16 @@ Feature: Get errors test @get-errors Scenario: Getting non-existent UAL Given I setup 4 nodes - And I wait for 2 seconds - + And I wait for 5 seconds + When I call Get directly on the node 1 with nonExistentUAL And I wait for latest resolve to finalize Then Latest Get operation finished with status: GetRouteError - + @get-errors Scenario: Getting invalid UAL Given I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Get directly on the node 1 with invalidUAL And I wait for latest resolve to finalize @@ -26,7 +26,7 @@ Feature: Get errors test Given I setup 4 nodes And I set R0 to be 1 And I set R1 to be 2 - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 1 with validAssertion And I wait for latest Publish to finalize @@ -38,7 +38,7 @@ Feature: Get errors test Given I setup 4 nodes And I set R0 to be 1 And I set R1 to be 2 - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 1 with validAssertion And I wait for latest Publish to finalize diff --git a/test/bdd/features/get.feature b/test/bdd/features/get.feature index c6e994b212..d11fcf2f30 100644 --- a/test/bdd/features/get.feature +++ b/test/bdd/features/get.feature @@ -9,7 +9,7 @@ Feature: Get asset states test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize @@ -29,7 +29,7 @@ Feature: Get asset states test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize @@ -49,7 +49,7 @@ Feature: Get asset states test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize diff --git a/test/bdd/features/publish-errors.feature b/test/bdd/features/publish-errors.feature index 47c62187fe..503f7040a3 100644 --- a/test/bdd/features/publish-errors.feature +++ b/test/bdd/features/publish-errors.feature @@ -5,8 +5,8 @@ Feature: Publish errors test @publish-errors Scenario: Publish on a node with minimum replication factor greater than the number of nodes - Given I setup 1 nodes - And I wait for 2 seconds + Given I setup 2 nodes + And I wait for 5 seconds When I call Publish on the node 1 with validAssertion And I wait for latest Publish to finalize @@ -15,7 +15,7 @@ Feature: Publish errors test @publish-errors Scenario: Publish a knowledge asset directly on the node Given I setup 1 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish directly on the node 1 with validPublishRequestBody And I wait for latest Publish to finalize diff --git a/test/bdd/features/publish.feature b/test/bdd/features/publish.feature index 0e28c9c9ed..3cdc2c54f4 100644 --- a/test/bdd/features/publish.feature +++ b/test/bdd/features/publish.feature @@ -8,7 +8,7 @@ Feature: Release related tests Given I set R0 to be 1 And I set R1 to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validAssertion And I wait for latest Publish to finalize diff --git a/test/bdd/features/update-errors.feature b/test/bdd/features/update-errors.feature index 6633eca3f2..c5e6c5659c 100644 --- a/test/bdd/features/update-errors.feature +++ b/test/bdd/features/update-errors.feature @@ -6,7 +6,7 @@ Feature: Update errors test @update-errors Scenario: Update knowledge asset that was not previously published Given I setup 1 node - And I wait for 2 seconds + And I wait for 5 seconds When I call Update directly on the node 1 with validUpdateRequestBody And I wait for latest Update to finalize diff --git a/test/bdd/features/update.feature b/test/bdd/features/update.feature index a70fd206ca..fb1d42a81e 100644 --- a/test/bdd/features/update.feature +++ b/test/bdd/features/update.feature @@ -9,7 +9,7 @@ Feature: Update asset test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize diff --git a/test/unit/mock/blockchain-module-manager-mock.js b/test/unit/mock/blockchain-module-manager-mock.js index 994f2c2d37..8236f00ec5 100644 --- a/test/unit/mock/blockchain-module-manager-mock.js +++ b/test/unit/mock/blockchain-module-manager-mock.js @@ -5,6 +5,10 @@ class BlockchainModuleManagerMock { return 20; } + getR1() { + return 8; + } + getR0() { return 3; } diff --git a/test/unit/mock/network-module-manager-mock.js b/test/unit/mock/network-module-manager-mock.js index 6c530fa249..8bc0c7b32f 100644 --- a/test/unit/mock/network-module-manager-mock.js +++ b/test/unit/mock/network-module-manager-mock.js @@ -1,3 +1,9 @@ -class NetworkModuleManagerMock {} +class NetworkModuleManagerMock { + getPeerId() { + return { + toB58String: () => 'myPeerId', + }; + } +} export default NetworkModuleManagerMock; diff --git a/test/unit/service/sharding-table-service.test.js b/test/unit/service/sharding-table-service.test.js index 373841881d..1bd0de441b 100644 --- a/test/unit/service/sharding-table-service.test.js +++ b/test/unit/service/sharding-table-service.test.js @@ -39,28 +39,56 @@ describe('Sharding table service test', async () => { expect(bidSuggestions).to.be.equal('3788323225298705400'); }); - it('Get bid suggestion, returns same token amount for size 1 Kb and size < 1 Kb', async () => { + it('Get bid suggestion, returns valid value for assertion size 1b and ask 1 wei', async () => { const epochsNumber = 5; const contentAssetStorageAddress = '0xABd59A9aa71847F499d624c492d3903dA953d67a'; const firstAssertionId = '0xb44062de45333119471934bc0340c05ff09c0b463392384bc2030cd0a20c334b'; const hashFunctionId = 1; - const bidSuggestion1Kb = await shardingTableService.getBidSuggestion( + const askInWei = '0.000000000000000001'; + const peers = shardingTableService.repositoryModuleManager.getAllPeerRecords(); + shardingTableService.repositoryModuleManager.getAllPeerRecords = () => { + peers.forEach((peer) => { + // eslint-disable-next-line no-param-reassign + peer.ask = askInWei; + }); + return peers; + }; + const bidSuggestion1B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, - BYTES_IN_KILOBYTE, + 1, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); - const bidSuggestion1B = await shardingTableService.getBidSuggestion( + expect(bidSuggestion1B).to.be.equal('15'); + const bidSuggestion10B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, - 1, + 10, + contentAssetStorageAddress, + firstAssertionId, + hashFunctionId, + ); + expect(bidSuggestion10B).to.be.equal('15'); + const bidSuggestion1024B = await shardingTableService.getBidSuggestion( + 'ganache', + epochsNumber, + 1024, + contentAssetStorageAddress, + firstAssertionId, + hashFunctionId, + ); + expect(bidSuggestion1024B).to.be.equal('15'); + const bidSuggestion2048B = await shardingTableService.getBidSuggestion( + 'ganache', + epochsNumber, + 2048, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); - expect(bidSuggestion1B).to.be.equal(bidSuggestion1Kb); + expect(bidSuggestion2048B).to.be.equal('30'); }); }); From ee06b74ab1eea90b3f7aa8af968cba7294f21e9d Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Thu, 17 Aug 2023 12:07:45 +0200 Subject: [PATCH 5/5] version bump (#2669) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a17d1b1ea5..5ce51dfe14 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.12", + "version": "6.0.13", "description": "OTNode V6", "main": "index.js", "type": "module",