From 0210bf3acd45c64dddd2849b9cdebf0c55a85119 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Fri, 22 Nov 2024 02:12:27 -0800 Subject: [PATCH 01/29] Create influx.js --- .../implementation/influx/influx.js | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/modules/repository/implementation/influx/influx.js diff --git a/src/modules/repository/implementation/influx/influx.js b/src/modules/repository/implementation/influx/influx.js new file mode 100644 index 000000000..adb42ce71 --- /dev/null +++ b/src/modules/repository/implementation/influx/influx.js @@ -0,0 +1,88 @@ +import { InfluxDB, Point } from '@influxdata/influxdb-client'; +import dotenv from 'dotenv'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import fs from 'fs'; + +// Dynamically resolve the directory +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Dynamically locate and load the `.env` file +function setEnvParameters() { + const projectRoot = path.resolve(__dirname, '../../../..'); // Adjust this path if needed + const realRoot = fs.realpathSync(projectRoot); // Resolve symbolic link to real path + const envPath = path.join(realRoot, '.env'); // Append `.env` to resolved path + + // Check if the `.env` file exists + if (fs.existsSync(envPath)) { + //console.log(`Loading environment variables from: ${envPath}`); + dotenv.config({ path: envPath }); // Load the environment variables + } else { + console.error(`.env file not found at: ${envPath}`); + process.exit(1); // Exit if `.env` is not found + } +} + +// Call `setEnvParameters` to load the environment variables +setEnvParameters(); + +// Preconfigure the InfluxDB variables using environment variables +const influxDBUrl = process.env.INFLUXDB_URL; +const influxDBToken = process.env.INFLUXDB_TOKEN; +const influxDBOrg = process.env.INFLUXDB_ORG; +const influxDBBucket = process.env.INFLUXDB_BUCKET; + +// Check if required variables are present +if (!influxDBUrl || !influxDBToken || !influxDBOrg || !influxDBBucket) { + console.error('One or more required environment variables are missing:'); + console.error({ + INFLUXDB_URL: influxDBUrl, + INFLUXDB_TOKEN: influxDBToken, + INFLUXDB_ORG: influxDBOrg, + INFLUXDB_BUCKET: influxDBBucket, + }); + process.exit(1); // Exit the script if variables are missing +} + +// Initialize the InfluxDB client (use the imported class here) +const influxDBClient = new InfluxDB({ + url: influxDBUrl, + token: influxDBToken, +}); + +// Create the write API +const writeApi = influxDBClient.getWriteApi(influxDBOrg, influxDBBucket, 'ns'); + +// Function to send event telemetry to InfluxDB +export async function sendEventTelemetry( + operationId, + blockchainId, + name, + timestamp, + value1 = null, + value2 = null, + value3 = null +) { + try { + const point = new Point('event') + .tag('operationId', operationId) + .tag('blockchainId', blockchainId) + .stringField('name', name) + .intField('timestamp', timestamp) + .stringField('value1', value1 !== null ? value1.toString() : '') + .stringField('value2', value2 !== null ? value2.toString() : '') + .stringField('value3', value3 !== null ? value3.toString() : ''); + + writeApi.writePoint(point); + await writeApi.flush(); + //console.log('Event telemetry logged to InfluxDB'); + } catch (err) { + console.error('Error sending telemetry to InfluxDB:', err); + throw err; + } +} + +export default { + sendEventTelemetry, +}; From 92273891103578f447c59b10b9bb846128a79304 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Fri, 22 Nov 2024 02:21:24 -0800 Subject: [PATCH 02/29] Update repository-module-manager.js --- src/modules/repository/repository-module-manager.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index e6dba2989..1c6fd645d 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -1,4 +1,5 @@ import BaseModuleManager from '../base-module-manager.js'; +import influx from './implementation/influx/influx.js'; class RepositoryModuleManager extends BaseModuleManager { getName() { @@ -217,7 +218,7 @@ class RepositoryModuleManager extends BaseModuleManager { value2 = null, value3 = null, ) { - return this.getRepository('event').createEventRecord( + return influx.sendEventTelemetry( operationId, blockchainId, name, @@ -225,7 +226,7 @@ class RepositoryModuleManager extends BaseModuleManager { value1, value2, value3, - ); + ); } async getUnpublishedEvents() { From 72ec930e1f2d6cefc9a371addcab1e3e94016381 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Fri, 22 Nov 2024 02:22:40 -0800 Subject: [PATCH 03/29] Update package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 53b0c31e4..d76680bfb 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "@ethersproject/bytes": "^5.7.0", "@ethersproject/hash": "^5.7.0", "@ethersproject/wallet": "^5.7.0", + "@influxdata/influxdb-client": "^1.35.0", "@polkadot/api": "^9.3.2", "@polkadot/keyring": "^10.1.7", "@polkadot/util": "^10.1.7", From 7310a0d46c021a89b39d50e04e1f68c501acf164 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Fri, 22 Nov 2024 02:28:37 -0800 Subject: [PATCH 04/29] fixed the .env variables --- src/modules/repository/implementation/influx/influx.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/repository/implementation/influx/influx.js b/src/modules/repository/implementation/influx/influx.js index adb42ce71..886dfd10b 100644 --- a/src/modules/repository/implementation/influx/influx.js +++ b/src/modules/repository/implementation/influx/influx.js @@ -10,7 +10,7 @@ const __dirname = path.dirname(__filename); // Dynamically locate and load the `.env` file function setEnvParameters() { - const projectRoot = path.resolve(__dirname, '../../../..'); // Adjust this path if needed + const projectRoot = path.resolve(__dirname, '../../../../..'); // Adjust this path if needed const realRoot = fs.realpathSync(projectRoot); // Resolve symbolic link to real path const envPath = path.join(realRoot, '.env'); // Append `.env` to resolved path From 8738c1432cf5fc0b650a36bed5012f1863ed4344 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 07:33:12 -0800 Subject: [PATCH 05/29] Create telemetry-quest.js --- .../implementation/telemetry-quest.js | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/modules/telemetry/implementation/telemetry-quest.js diff --git a/src/modules/telemetry/implementation/telemetry-quest.js b/src/modules/telemetry/implementation/telemetry-quest.js new file mode 100644 index 000000000..8d8bdec33 --- /dev/null +++ b/src/modules/telemetry/implementation/telemetry-quest.js @@ -0,0 +1,53 @@ +import { Sender } from '@questdb/nodejs-client'; + +async function sendEventTelemetry( + operationId = '', + blockchainId = '', + name = '', + timestamp, // Accept timestamp as Unix timestamp (milliseconds) + value1 = null, + value2 = null, + value3 = null +) { + try { + // Correct format for configuration string + const configString = 'http::addr=localhost:10000'; // Adjust with your QuestDB address/port + + // Create sender instance with the proper config string + const sender = Sender.fromConfig(configString); + + // Set the table to use + const table = sender.table('event'); // Ensure the table name matches your DB + + // Define the columns in the table + table.symbol('operationId', operationId || 'NULL'); + table.symbol('blockchainId', blockchainId || 'NULL'); + table.symbol('name', name || 'NULL'); + + // Define timestamp as the Unix timestamp in milliseconds + table.timestampColumn('timestamp', timestamp || Date.now()); // Default to current timestamp in milliseconds + + // Insert value columns if they are provided + if (value1 !== null) table.string('value1', value1); + if (value2 !== null) table.string('value2', value2); + if (value3 !== null) table.string('value3', value3); + + // Send the telemetry data + await table.at(Date.now(), 'ms'); // Sends data with the current timestamp + + // Flush the buffer to send the data to QuestDB + await sender.flush(); + + // Close the sender connection after sending the data + await sender.close(); + + console.log('Event telemetry logged to QuestDB'); + } catch (err) { + console.error('Error sending telemetry to QuestDB:', err); + throw err; // Rethrow error for further handling + } +} + +export default { + sendEventTelemetry, +}; From d55473cad596e3b198b5e43204a8fbae390009b0 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 07:39:20 -0800 Subject: [PATCH 06/29] Update telemetry-module-manager.js --- .../telemetry/telemetry-module-manager.js | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index f5ef04377..f13158420 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -1,5 +1,5 @@ import BaseModuleManager from '../base-module-manager.js'; - +import questdb from './implementation/telemetry-quest.js'; class TelemetryModuleManager extends BaseModuleManager { constructor(ctx) { super(ctx); @@ -24,6 +24,25 @@ class TelemetryModuleManager extends BaseModuleManager { return this.getImplementation().module.sendTelemetryData(nodeData, events); } } + async createEventRecord( + operationId, + blockchainId, + name, + timestamp, + value1 = null, + value2 = null, + value3 = null, + ) { + return questdb.sendEventTelemetry( + operationId, + blockchainId, + name, + timestamp, + value1, + value2, + value3, + ); + } } export default TelemetryModuleManager; From 5253643aa2326db6b0e2da0b1226cab8fef59772 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 07:41:49 -0800 Subject: [PATCH 07/29] Update ot-node.js updated to send events directly to telemetry module --- ot-node.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ot-node.js b/ot-node.js index e454613d6..bef897aaf 100644 --- a/ot-node.js +++ b/ot-node.js @@ -350,9 +350,8 @@ class OTNode { startTelemetryModule() { const telemetryModuleManager = this.container.resolve('telemetryModuleManager'); - const repositoryModuleManager = this.container.resolve('repositoryModuleManager'); telemetryModuleManager.listenOnEvents((eventData) => { - repositoryModuleManager.createEventRecord( + telemetryModuleManager.createEventRecord( eventData.operationId, eventData.blockchainId, eventData.lastEvent, From 3efe8bcaab5b06c37439c61698323db0fa059749 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 09:43:30 -0800 Subject: [PATCH 08/29] Update telemetry-quest.js --- .../implementation/telemetry-quest.js | 84 ++++++++----------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/src/modules/telemetry/implementation/telemetry-quest.js b/src/modules/telemetry/implementation/telemetry-quest.js index 8d8bdec33..10a681fde 100644 --- a/src/modules/telemetry/implementation/telemetry-quest.js +++ b/src/modules/telemetry/implementation/telemetry-quest.js @@ -1,53 +1,43 @@ import { Sender } from '@questdb/nodejs-client'; -async function sendEventTelemetry( - operationId = '', - blockchainId = '', - name = '', - timestamp, // Accept timestamp as Unix timestamp (milliseconds) - value1 = null, - value2 = null, - value3 = null -) { - try { - // Correct format for configuration string - const configString = 'http::addr=localhost:10000'; // Adjust with your QuestDB address/port - - // Create sender instance with the proper config string - const sender = Sender.fromConfig(configString); - - // Set the table to use - const table = sender.table('event'); // Ensure the table name matches your DB - - // Define the columns in the table - table.symbol('operationId', operationId || 'NULL'); - table.symbol('blockchainId', blockchainId || 'NULL'); - table.symbol('name', name || 'NULL'); - - // Define timestamp as the Unix timestamp in milliseconds - table.timestampColumn('timestamp', timestamp || Date.now()); // Default to current timestamp in milliseconds - - // Insert value columns if they are provided - if (value1 !== null) table.string('value1', value1); - if (value2 !== null) table.string('value2', value2); - if (value3 !== null) table.string('value3', value3); - - // Send the telemetry data - await table.at(Date.now(), 'ms'); // Sends data with the current timestamp - - // Flush the buffer to send the data to QuestDB - await sender.flush(); - - // Close the sender connection after sending the data - await sender.close(); +class TelemetryQuest { + async initialize(config, logger) { + this.config = config; // Store configuration if needed + this.logger = logger; // Assign logger passed from caller + } - console.log('Event telemetry logged to QuestDB'); - } catch (err) { - console.error('Error sending telemetry to QuestDB:', err); - throw err; // Rethrow error for further handling + async sendEventTelemetry( + operationId = '', + blockchainId = '', + name = '', + timestamp, // Accept timestamp as Unix timestamp (milliseconds) + value1 = null, + value2 = null, + value3 = null + ) { + try { + const configString = 'http::addr=localhost:10000'; // Adjust with your QuestDB address/port + const sender = Sender.fromConfig(configString); + const table = sender.table('event'); // Ensure the table name matches your DB + + table.symbol('operationId', operationId || 'NULL'); + table.symbol('blockchainId', blockchainId || 'NULL'); + table.symbol('name', name || 'NULL'); + table.timestampColumn('timestamp', timestamp || Date.now()); // Default to current timestamp in milliseconds + if (value1 !== null) table.string('value1', value1); + if (value2 !== null) table.string('value2', value2); + if (value3 !== null) table.string('value3', value3); + + await table.at(Date.now(), 'ms'); // Sends data with the current timestamp + await sender.flush(); + await sender.close(); + + this.logger.info('Event telemetry successfully logged to QuestDB'); + } catch (err) { + this.logger.error(`Error sending telemetry to QuestDB: ${err.message}`); + throw err; // Rethrow error for further handling + } } } -export default { - sendEventTelemetry, -}; +export default TelemetryQuest; From 7140645bd66c8563471d21045a97ee50012fcb86 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 09:49:52 -0800 Subject: [PATCH 09/29] Update telemetry-module-manager.js --- .../telemetry/telemetry-module-manager.js | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index f13158420..24dadbbbf 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -1,5 +1,5 @@ import BaseModuleManager from '../base-module-manager.js'; -import questdb from './implementation/telemetry-quest.js'; +import TelemetryQuest from './implementation/telemetry-quest.js'; class TelemetryModuleManager extends BaseModuleManager { constructor(ctx) { super(ctx); @@ -33,15 +33,17 @@ class TelemetryModuleManager extends BaseModuleManager { value2 = null, value3 = null, ) { - return questdb.sendEventTelemetry( - operationId, - blockchainId, - name, - timestamp, - value1, - value2, - value3, - ); + if (this.initialized) { + return this.telemetryQuest.sendEventTelemetry( + operationId, + blockchainId, + name, + timestamp, + value1, + value2, + value3, + ); + } } } From c51adac87c48b675d9b3f25fa04e2400a39b5aa6 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:07:28 -0800 Subject: [PATCH 10/29] Rename telemetry-quest.js to local-telemetry.js --- .../implementation/{telemetry-quest.js => local-telemetry.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/modules/telemetry/implementation/{telemetry-quest.js => local-telemetry.js} (100%) diff --git a/src/modules/telemetry/implementation/telemetry-quest.js b/src/modules/telemetry/implementation/local-telemetry.js similarity index 100% rename from src/modules/telemetry/implementation/telemetry-quest.js rename to src/modules/telemetry/implementation/local-telemetry.js From 746fa06fab247a1f0f1009157b6c41f3ca6aa9a8 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:08:03 -0800 Subject: [PATCH 11/29] Update telemetry-module-manager.js --- src/modules/telemetry/telemetry-module-manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index 24dadbbbf..8e42b08ee 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -1,5 +1,5 @@ import BaseModuleManager from '../base-module-manager.js'; -import TelemetryQuest from './implementation/telemetry-quest.js'; +import TelemetryQuest from './implementation/local-telemetry.js'; class TelemetryModuleManager extends BaseModuleManager { constructor(ctx) { super(ctx); From 0523e81fdd63971075e7b4734f332f6d96147e50 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:23:53 -0800 Subject: [PATCH 12/29] Update config.json --- config/config.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/config/config.json b/config/config.json index b516c06fb..acf4583c7 100644 --- a/config/config.json +++ b/config/config.json @@ -158,6 +158,10 @@ "sendTelemetryData": false, "signalingServerUrl": "null" } + }, + "local-telemetry": { + "enabled": false, + "package": "./telemetry/implementation/local-telemetry.js" } } } @@ -326,6 +330,10 @@ "sendTelemetryData": false, "signalingServerUrl": "null" } + }, + "local-telemetry": { + "enabled": false, + "package": "./telemetry/implementation/local-telemetry.js" } } } @@ -494,6 +502,10 @@ "sendTelemetryData": true, "signalingServerUrl": "https://v8-testnet-signaling.origin-trail.network/signal" } + }, + "local-telemetry": { + "enabled": false, + "package": "./telemetry/implementation/local-telemetry.js" } } } @@ -661,6 +673,10 @@ "sendTelemetryData": true, "signalingServerUrl": "https://devnet-signaling.origin-trail.network/signal" } + }, + "local-telemetry": { + "enabled": false, + "package": "./telemetry/implementation/local-telemetry.js" } } } @@ -851,6 +867,10 @@ "sendTelemetryData": true, "signalingServerUrl": "https://mainnet-signaling.origin-trail.network/signal" } + }, + "local-telemetry": { + "enabled": false, + "package": "./telemetry/implementation/local-telemetry.js" } } } From da1dc98601ec18c36764ae18b5ff9ddc346bd30f Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Thu, 28 Nov 2024 06:42:55 -0800 Subject: [PATCH 13/29] Update telemetry-module-manager.js --- src/modules/telemetry/telemetry-module-manager.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index 8e42b08ee..f6e8cc718 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -1,9 +1,12 @@ import BaseModuleManager from '../base-module-manager.js'; import TelemetryQuest from './implementation/local-telemetry.js'; + class TelemetryModuleManager extends BaseModuleManager { constructor(ctx) { super(ctx); this.eventEmitter = ctx.eventEmitter; + this.telemetryQuest = new TelemetryQuest(); + this.telemetryQuest.initialize(ctx.config.modules.telemetry, ctx.logger); } getName() { @@ -24,6 +27,7 @@ class TelemetryModuleManager extends BaseModuleManager { return this.getImplementation().module.sendTelemetryData(nodeData, events); } } + async createEventRecord( operationId, blockchainId, From 79a86657053c6c855844e53374aff16713ba3dde Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:46:18 -0800 Subject: [PATCH 14/29] Update local-telemetry.js --- src/modules/telemetry/implementation/local-telemetry.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/modules/telemetry/implementation/local-telemetry.js b/src/modules/telemetry/implementation/local-telemetry.js index 10a681fde..c090a01f2 100644 --- a/src/modules/telemetry/implementation/local-telemetry.js +++ b/src/modules/telemetry/implementation/local-telemetry.js @@ -23,10 +23,10 @@ class TelemetryQuest { table.symbol('operationId', operationId || 'NULL'); table.symbol('blockchainId', blockchainId || 'NULL'); table.symbol('name', name || 'NULL'); - table.timestampColumn('timestamp', timestamp || Date.now()); // Default to current timestamp in milliseconds - if (value1 !== null) table.string('value1', value1); - if (value2 !== null) table.string('value2', value2); - if (value3 !== null) table.string('value3', value3); + if (value1 !== null) table.symbol('value1', value1); + if (value2 !== null) table.symbol('value2', value2); + if (value3 !== null) table.symbol('value3', value3); + table.timestampColumn('timestamp', timestamp * 1000); await table.at(Date.now(), 'ms'); // Sends data with the current timestamp await sender.flush(); From f89ef35999d2fc9eb0aa403e54a964dd5ecced89 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:35:15 -0800 Subject: [PATCH 15/29] Update local-telemetry.js --- .../telemetry/implementation/local-telemetry.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/modules/telemetry/implementation/local-telemetry.js b/src/modules/telemetry/implementation/local-telemetry.js index c090a01f2..814cc759c 100644 --- a/src/modules/telemetry/implementation/local-telemetry.js +++ b/src/modules/telemetry/implementation/local-telemetry.js @@ -2,23 +2,23 @@ import { Sender } from '@questdb/nodejs-client'; class TelemetryQuest { async initialize(config, logger) { - this.config = config; // Store configuration if needed - this.logger = logger; // Assign logger passed from caller + this.config = config; + this.logger = logger; } async sendEventTelemetry( operationId = '', blockchainId = '', name = '', - timestamp, // Accept timestamp as Unix timestamp (milliseconds) + timestamp, value1 = null, value2 = null, value3 = null ) { try { - const configString = 'http::addr=localhost:10000'; // Adjust with your QuestDB address/port + const configString = 'http::addr=localhost:10000'; const sender = Sender.fromConfig(configString); - const table = sender.table('event'); // Ensure the table name matches your DB + const table = sender.table('event'); table.symbol('operationId', operationId || 'NULL'); table.symbol('blockchainId', blockchainId || 'NULL'); @@ -28,14 +28,14 @@ class TelemetryQuest { if (value3 !== null) table.symbol('value3', value3); table.timestampColumn('timestamp', timestamp * 1000); - await table.at(Date.now(), 'ms'); // Sends data with the current timestamp + await table.at(Date.now(), 'ms'); await sender.flush(); await sender.close(); this.logger.info('Event telemetry successfully logged to QuestDB'); } catch (err) { this.logger.error(`Error sending telemetry to QuestDB: ${err.message}`); - throw err; // Rethrow error for further handling + throw err; } } } From d6b72989c87115d680d3c4963836e0b9268bad51 Mon Sep 17 00:00:00 2001 From: Samuel Wamala <35219064+swamala@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:42:59 -0800 Subject: [PATCH 16/29] Delete src/modules/repository/implementation/influx directory --- .../implementation/influx/influx.js | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 src/modules/repository/implementation/influx/influx.js diff --git a/src/modules/repository/implementation/influx/influx.js b/src/modules/repository/implementation/influx/influx.js deleted file mode 100644 index 886dfd10b..000000000 --- a/src/modules/repository/implementation/influx/influx.js +++ /dev/null @@ -1,88 +0,0 @@ -import { InfluxDB, Point } from '@influxdata/influxdb-client'; -import dotenv from 'dotenv'; -import path from 'path'; -import { fileURLToPath } from 'url'; -import fs from 'fs'; - -// Dynamically resolve the directory -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -// Dynamically locate and load the `.env` file -function setEnvParameters() { - const projectRoot = path.resolve(__dirname, '../../../../..'); // Adjust this path if needed - const realRoot = fs.realpathSync(projectRoot); // Resolve symbolic link to real path - const envPath = path.join(realRoot, '.env'); // Append `.env` to resolved path - - // Check if the `.env` file exists - if (fs.existsSync(envPath)) { - //console.log(`Loading environment variables from: ${envPath}`); - dotenv.config({ path: envPath }); // Load the environment variables - } else { - console.error(`.env file not found at: ${envPath}`); - process.exit(1); // Exit if `.env` is not found - } -} - -// Call `setEnvParameters` to load the environment variables -setEnvParameters(); - -// Preconfigure the InfluxDB variables using environment variables -const influxDBUrl = process.env.INFLUXDB_URL; -const influxDBToken = process.env.INFLUXDB_TOKEN; -const influxDBOrg = process.env.INFLUXDB_ORG; -const influxDBBucket = process.env.INFLUXDB_BUCKET; - -// Check if required variables are present -if (!influxDBUrl || !influxDBToken || !influxDBOrg || !influxDBBucket) { - console.error('One or more required environment variables are missing:'); - console.error({ - INFLUXDB_URL: influxDBUrl, - INFLUXDB_TOKEN: influxDBToken, - INFLUXDB_ORG: influxDBOrg, - INFLUXDB_BUCKET: influxDBBucket, - }); - process.exit(1); // Exit the script if variables are missing -} - -// Initialize the InfluxDB client (use the imported class here) -const influxDBClient = new InfluxDB({ - url: influxDBUrl, - token: influxDBToken, -}); - -// Create the write API -const writeApi = influxDBClient.getWriteApi(influxDBOrg, influxDBBucket, 'ns'); - -// Function to send event telemetry to InfluxDB -export async function sendEventTelemetry( - operationId, - blockchainId, - name, - timestamp, - value1 = null, - value2 = null, - value3 = null -) { - try { - const point = new Point('event') - .tag('operationId', operationId) - .tag('blockchainId', blockchainId) - .stringField('name', name) - .intField('timestamp', timestamp) - .stringField('value1', value1 !== null ? value1.toString() : '') - .stringField('value2', value2 !== null ? value2.toString() : '') - .stringField('value3', value3 !== null ? value3.toString() : ''); - - writeApi.writePoint(point); - await writeApi.flush(); - //console.log('Event telemetry logged to InfluxDB'); - } catch (err) { - console.error('Error sending telemetry to InfluxDB:', err); - throw err; - } -} - -export default { - sendEventTelemetry, -}; From 7a39de99ab30875e8651aa5dd5b35c5a5c7343a7 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Mon, 2 Dec 2024 14:03:41 +0100 Subject: [PATCH 17/29] Telemetry refactor --- config/config.json | 86 ++++++++----------- ot-node.js | 2 +- package-lock.json | 11 +++ package.json | 2 +- .../repositories/event-repository.js | 12 --- .../repository/repository-module-manager.js | 21 ----- .../telemetry/implementation/ot-telemetry.js | 29 ------- ...{local-telemetry.js => quest-telemetry.js} | 30 ++++--- .../telemetry/telemetry-module-manager.js | 23 +---- 9 files changed, 69 insertions(+), 147 deletions(-) delete mode 100644 src/modules/telemetry/implementation/ot-telemetry.js rename src/modules/telemetry/implementation/{local-telemetry.js => quest-telemetry.js} (59%) diff --git a/config/config.json b/config/config.json index acf4583c7..bfd11bb45 100644 --- a/config/config.json +++ b/config/config.json @@ -151,17 +151,13 @@ "telemetry": { "enabled": true, "implementation": { - "ot-telemetry": { + "quest-telemetry": { "enabled": true, - "package": "./telemetry/implementation/ot-telemetry.js", + "package": "./telemetry/implementation/quest-telemetry.js", "config": { - "sendTelemetryData": false, - "signalingServerUrl": "null" - } - }, - "local-telemetry": { - "enabled": false, - "package": "./telemetry/implementation/local-telemetry.js" + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } } } @@ -323,17 +319,15 @@ "telemetry": { "enabled": true, "implementation": { - "ot-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/ot-telemetry.js", - "config": { - "sendTelemetryData": false, - "signalingServerUrl": "null" + "implementation": { + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } - }, - "local-telemetry": { - "enabled": false, - "package": "./telemetry/implementation/local-telemetry.js" } } } @@ -495,17 +489,15 @@ "telemetry": { "enabled": true, "implementation": { - "ot-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/ot-telemetry.js", - "config": { - "sendTelemetryData": true, - "signalingServerUrl": "https://v8-testnet-signaling.origin-trail.network/signal" + "implementation": { + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } - }, - "local-telemetry": { - "enabled": false, - "package": "./telemetry/implementation/local-telemetry.js" } } } @@ -666,17 +658,15 @@ "telemetry": { "enabled": true, "implementation": { - "ot-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/ot-telemetry.js", - "config": { - "sendTelemetryData": true, - "signalingServerUrl": "https://devnet-signaling.origin-trail.network/signal" + "implementation": { + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } - }, - "local-telemetry": { - "enabled": false, - "package": "./telemetry/implementation/local-telemetry.js" } } } @@ -860,17 +850,15 @@ "telemetry": { "enabled": true, "implementation": { - "ot-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/ot-telemetry.js", - "config": { - "sendTelemetryData": true, - "signalingServerUrl": "https://mainnet-signaling.origin-trail.network/signal" + "implementation": { + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } - }, - "local-telemetry": { - "enabled": false, - "package": "./telemetry/implementation/local-telemetry.js" } } } diff --git a/ot-node.js b/ot-node.js index bef897aaf..ca38e5d57 100644 --- a/ot-node.js +++ b/ot-node.js @@ -353,9 +353,9 @@ class OTNode { telemetryModuleManager.listenOnEvents((eventData) => { telemetryModuleManager.createEventRecord( eventData.operationId, + eventData.timestamp, eventData.blockchainId, eventData.lastEvent, - eventData.timestamp, eventData.value1, eventData.value2, eventData.value3, diff --git a/package-lock.json b/package-lock.json index b0cba1c9e..6464e1127 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@polkadot/keyring": "^10.1.7", "@polkadot/util": "^10.1.7", "@polkadot/util-crypto": "^10.1.7", + "@questdb/nodejs-client": "^3.0.0", "app-root-path": "^3.1.0", "assertion-tools": "^2.0.2", "async": "^3.2.4", @@ -4835,6 +4836,11 @@ "version": "1.1.0", "license": "BSD-3-Clause" }, + "node_modules/@questdb/nodejs-client": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@questdb/nodejs-client/-/nodejs-client-3.0.0.tgz", + "integrity": "sha512-lBKd732rRpS/pqyWgCJFVXi9N1YoWDAUZzp6BngBVxH92As/NDXigZmCYKQVKpAEGYx14606CH5AoJ23qf3oqA==" + }, "node_modules/@rdfjs/types": { "version": "1.1.0", "license": "MIT", @@ -24807,6 +24813,11 @@ "@protobufjs/utf8": { "version": "1.1.0" }, + "@questdb/nodejs-client": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@questdb/nodejs-client/-/nodejs-client-3.0.0.tgz", + "integrity": "sha512-lBKd732rRpS/pqyWgCJFVXi9N1YoWDAUZzp6BngBVxH92As/NDXigZmCYKQVKpAEGYx14606CH5AoJ23qf3oqA==" + }, "@rdfjs/types": { "version": "1.1.0", "requires": { diff --git a/package.json b/package.json index d76680bfb..bf483d956 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "@ethersproject/bytes": "^5.7.0", "@ethersproject/hash": "^5.7.0", "@ethersproject/wallet": "^5.7.0", - "@influxdata/influxdb-client": "^1.35.0", + "@questdb/nodejs-client": "^3.0.0", "@polkadot/api": "^9.3.2", "@polkadot/keyring": "^10.1.7", "@polkadot/util": "^10.1.7", diff --git a/src/modules/repository/implementation/sequelize/repositories/event-repository.js b/src/modules/repository/implementation/sequelize/repositories/event-repository.js index 276f83ce5..8304bfc66 100644 --- a/src/modules/repository/implementation/sequelize/repositories/event-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/event-repository.js @@ -11,18 +11,6 @@ class EventRepository { this.model = models.event; } - async createEventRecord(operationId, blockchainId, name, timestamp, value1, value2, value3) { - return this.model.create({ - operationId, - blockchainId, - name, - timestamp, - value1, - value2, - value3, - }); - } - async getUnpublishedEvents() { // events without COMPLETE/FAILED status which are older than 30min // are also considered finished diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 1c6fd645d..405a5bf02 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -1,5 +1,4 @@ import BaseModuleManager from '../base-module-manager.js'; -import influx from './implementation/influx/influx.js'; class RepositoryModuleManager extends BaseModuleManager { getName() { @@ -209,26 +208,6 @@ class RepositoryModuleManager extends BaseModuleManager { return this.getRepository('shard').cleanShardingTable(blockchainId); } - async createEventRecord( - operationId, - blockchainId, - name, - timestamp, - value1 = null, - value2 = null, - value3 = null, - ) { - return influx.sendEventTelemetry( - operationId, - blockchainId, - name, - timestamp, - value1, - value2, - value3, - ); - } - async getUnpublishedEvents() { return this.getRepository('event').getUnpublishedEvents(); } diff --git a/src/modules/telemetry/implementation/ot-telemetry.js b/src/modules/telemetry/implementation/ot-telemetry.js deleted file mode 100644 index c4460c757..000000000 --- a/src/modules/telemetry/implementation/ot-telemetry.js +++ /dev/null @@ -1,29 +0,0 @@ -import axios from 'axios'; - -class OTTelemetry { - async initialize(config, logger) { - this.config = config; - this.logger = logger; - } - - listenOnEvents(eventEmitter, onEventReceived) { - return eventEmitter.on('operation_status_changed', onEventReceived); - } - - async sendTelemetryData(nodeData, events) { - const signalingMessage = { nodeData, events }; - const config = { - method: 'post', - url: this.config.signalingServerUrl, - headers: { - 'Content-Type': 'application/json', - }, - data: JSON.stringify(signalingMessage), - }; - const response = await axios(config); - const isSuccess = response.status === 200; - return isSuccess; - } -} - -export default OTTelemetry; diff --git a/src/modules/telemetry/implementation/local-telemetry.js b/src/modules/telemetry/implementation/quest-telemetry.js similarity index 59% rename from src/modules/telemetry/implementation/local-telemetry.js rename to src/modules/telemetry/implementation/quest-telemetry.js index 814cc759c..4f6460483 100644 --- a/src/modules/telemetry/implementation/local-telemetry.js +++ b/src/modules/telemetry/implementation/quest-telemetry.js @@ -1,24 +1,27 @@ import { Sender } from '@questdb/nodejs-client'; -class TelemetryQuest { +class QuestTelemetry { async initialize(config, logger) { - this.config = config; - this.logger = logger; + this.config = config; + this.logger = logger; } - async sendEventTelemetry( - operationId = '', + listenOnEvents(eventEmitter, onEventReceived) { + return eventEmitter.on('operation_status_changed', onEventReceived); + } + + async sendTelemetryData( + operationId, + timestamp, blockchainId = '', name = '', - timestamp, value1 = null, value2 = null, - value3 = null + value3 = null, ) { try { - const configString = 'http::addr=localhost:10000'; - const sender = Sender.fromConfig(configString); - const table = sender.table('event'); + const sender = Sender.fromConfig(this.config.ip_endpoint); + const table = sender.table('event'); table.symbol('operationId', operationId || 'NULL'); table.symbol('blockchainId', blockchainId || 'NULL'); @@ -28,16 +31,15 @@ class TelemetryQuest { if (value3 !== null) table.symbol('value3', value3); table.timestampColumn('timestamp', timestamp * 1000); - await table.at(Date.now(), 'ms'); + await table.at(Date.now(), 'ms'); await sender.flush(); await sender.close(); - this.logger.info('Event telemetry successfully logged to QuestDB'); + this.logger.info('Event telemetry successfully sent to QuestDB'); } catch (err) { this.logger.error(`Error sending telemetry to QuestDB: ${err.message}`); - throw err; } } } -export default TelemetryQuest; +export default QuestTelemetry; diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index f6e8cc718..63c6aab6c 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -1,12 +1,9 @@ import BaseModuleManager from '../base-module-manager.js'; -import TelemetryQuest from './implementation/local-telemetry.js'; class TelemetryModuleManager extends BaseModuleManager { constructor(ctx) { super(ctx); this.eventEmitter = ctx.eventEmitter; - this.telemetryQuest = new TelemetryQuest(); - this.telemetryQuest.initialize(ctx.config.modules.telemetry, ctx.logger); } getName() { @@ -22,27 +19,13 @@ class TelemetryModuleManager extends BaseModuleManager { } } - async sendTelemetryData(nodeData, events) { + async sendTelemetryData(operationId, timestamp, blockchainId, name, value1, value2, value3) { if (this.initialized) { - return this.getImplementation().module.sendTelemetryData(nodeData, events); - } - } - - async createEventRecord( - operationId, - blockchainId, - name, - timestamp, - value1 = null, - value2 = null, - value3 = null, - ) { - if (this.initialized) { - return this.telemetryQuest.sendEventTelemetry( + return this.getImplementation().module.sendTelemetryData( operationId, + timestamp, blockchainId, name, - timestamp, value1, value2, value3, From 8d476b611f589ef6a2972d865a8edfd896777d6a Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Mon, 2 Dec 2024 14:24:09 +0100 Subject: [PATCH 18/29] PR suggesstiongs --- .../implementation/quest-telemetry.js | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/modules/telemetry/implementation/quest-telemetry.js b/src/modules/telemetry/implementation/quest-telemetry.js index 4f6460483..4becd23c4 100644 --- a/src/modules/telemetry/implementation/quest-telemetry.js +++ b/src/modules/telemetry/implementation/quest-telemetry.js @@ -4,6 +4,8 @@ class QuestTelemetry { async initialize(config, logger) { this.config = config; this.logger = logger; + this.sender = Sender.fromConfig(this.config.ip_endpoint); + this.table = this.sender.table('event'); } listenOnEvents(eventEmitter, onEventReceived) { @@ -20,20 +22,17 @@ class QuestTelemetry { value3 = null, ) { try { - const sender = Sender.fromConfig(this.config.ip_endpoint); - const table = sender.table('event'); + this.table.symbol('operationId', operationId || 'NULL'); + this.table.symbol('blockchainId', blockchainId || 'NULL'); + this.table.symbol('name', name || 'NULL'); + if (value1 !== null) this.table.symbol('value1', value1); + if (value2 !== null) this.table.symbol('value2', value2); + if (value3 !== null) this.table.symbol('value3', value3); + this.table.timestampColumn('timestamp', timestamp * 1000); - table.symbol('operationId', operationId || 'NULL'); - table.symbol('blockchainId', blockchainId || 'NULL'); - table.symbol('name', name || 'NULL'); - if (value1 !== null) table.symbol('value1', value1); - if (value2 !== null) table.symbol('value2', value2); - if (value3 !== null) table.symbol('value3', value3); - table.timestampColumn('timestamp', timestamp * 1000); - - await table.at(Date.now(), 'ms'); - await sender.flush(); - await sender.close(); + await this.table.at(Date.now(), 'ms'); + await this.sender.flush(); + await this.sender.close(); this.logger.info('Event telemetry successfully sent to QuestDB'); } catch (err) { From b1c7abc93efda011ff4e5d73e81b5ca0f82dd449 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Mon, 2 Dec 2024 15:53:12 +0100 Subject: [PATCH 19/29] Fix config mistake --- config/config.json | 91 +++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/config/config.json b/config/config.json index bfd11bb45..d184c1f11 100644 --- a/config/config.json +++ b/config/config.json @@ -95,7 +95,32 @@ "ot-blazegraph": { "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", - "config": {} + "config": { + "publicCurrent": { + "url": "http://localhost:9999", + "name": "kb", + "username": "admin", + "password": "" + }, + "publicHistory": { + "url": "http://localhost:9999", + "name": "public-history", + "username": "admin", + "password": "" + }, + "privateCurrent": { + "url": "http://localhost:9999", + "name": "private-current", + "username": "admin", + "password": "" + }, + "privateHistory": { + "url": "http://localhost:9999", + "name": "private-history", + "username": "admin", + "password": "" + } + } }, "ot-fuseki": { "enabled": false, @@ -319,15 +344,13 @@ "telemetry": { "enabled": true, "implementation": { - "implementation": { - "quest-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/quest-telemetry.js", - "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false - } - } + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } } } @@ -489,15 +512,13 @@ "telemetry": { "enabled": true, "implementation": { - "implementation": { - "quest-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/quest-telemetry.js", - "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false - } - } + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } } } @@ -658,15 +679,13 @@ "telemetry": { "enabled": true, "implementation": { - "implementation": { - "quest-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/quest-telemetry.js", - "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false - } - } + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } } } @@ -850,15 +869,13 @@ "telemetry": { "enabled": true, "implementation": { - "implementation": { - "quest-telemetry": { - "enabled": true, - "package": "./telemetry/implementation/quest-telemetry.js", - "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false - } - } + "quest-telemetry": { + "enabled": true, + "package": "./telemetry/implementation/quest-telemetry.js", + "config": { + "ip_endpoint": "http::addr=localhost:10000", + "sendTelemetryData": false + } } } } From 05ea019f6381963bc332588bba024b908f355b24 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Tue, 3 Dec 2024 10:08:53 +0100 Subject: [PATCH 20/29] Fix config --- config/config.json | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/config/config.json b/config/config.json index d184c1f11..7d06fb706 100644 --- a/config/config.json +++ b/config/config.json @@ -95,32 +95,7 @@ "ot-blazegraph": { "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", - "config": { - "publicCurrent": { - "url": "http://localhost:9999", - "name": "kb", - "username": "admin", - "password": "" - }, - "publicHistory": { - "url": "http://localhost:9999", - "name": "public-history", - "username": "admin", - "password": "" - }, - "privateCurrent": { - "url": "http://localhost:9999", - "name": "private-current", - "username": "admin", - "password": "" - }, - "privateHistory": { - "url": "http://localhost:9999", - "name": "private-history", - "username": "admin", - "password": "" - } - } + "config": {} }, "ot-fuseki": { "enabled": false, From fe19438fb5c74c411859c8b54aac6ef0ca69b431 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Tue, 3 Dec 2024 11:03:50 +0100 Subject: [PATCH 21/29] Fix wrong function name in ot-node --- ot-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ot-node.js b/ot-node.js index ca38e5d57..438e95f08 100644 --- a/ot-node.js +++ b/ot-node.js @@ -351,7 +351,7 @@ class OTNode { startTelemetryModule() { const telemetryModuleManager = this.container.resolve('telemetryModuleManager'); telemetryModuleManager.listenOnEvents((eventData) => { - telemetryModuleManager.createEventRecord( + telemetryModuleManager.sendTelemetryData( eventData.operationId, eventData.timestamp, eventData.blockchainId, From 4220d6cfe596e8443c2441de302e164df495d780 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Tue, 3 Dec 2024 11:47:42 +0100 Subject: [PATCH 22/29] Add sendTelemetryData check --- src/modules/telemetry/telemetry-module-manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index 63c6aab6c..430083904 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -20,7 +20,7 @@ class TelemetryModuleManager extends BaseModuleManager { } async sendTelemetryData(operationId, timestamp, blockchainId, name, value1, value2, value3) { - if (this.initialized) { + if (this.getImplementation().config.sendTelemetryData && this.initialized) { return this.getImplementation().module.sendTelemetryData( operationId, timestamp, From 80ea933f15b8e80136e4041fc1c57bb7ab0aced0 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Tue, 3 Dec 2024 12:43:32 +0100 Subject: [PATCH 23/29] Start telemetry in telemetry module manager --- ot-node.js | 16 ---------------- .../telemetry/telemetry-module-manager.js | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ot-node.js b/ot-node.js index 438e95f08..3be681c47 100644 --- a/ot-node.js +++ b/ot-node.js @@ -104,7 +104,6 @@ class OTNode { await this.initializeRouters(); await this.startNetworkModule(); - this.startTelemetryModule(); this.resumeCommandExecutor(); this.logger.info('Node is up and running!'); @@ -348,21 +347,6 @@ class OTNode { await networkModuleManager.start(); } - startTelemetryModule() { - const telemetryModuleManager = this.container.resolve('telemetryModuleManager'); - telemetryModuleManager.listenOnEvents((eventData) => { - telemetryModuleManager.sendTelemetryData( - eventData.operationId, - eventData.timestamp, - eventData.blockchainId, - eventData.lastEvent, - eventData.value1, - eventData.value2, - eventData.value3, - ); - }); - } - async initializeShardingTableService() { try { const shardingTableService = this.container.resolve('shardingTableService'); diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index 430083904..19b8e1182 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -10,6 +10,22 @@ class TelemetryModuleManager extends BaseModuleManager { return 'telemetry'; } + async initialize() { + await super.initialize(); + + this.listenOnEvents((eventData) => { + this.sendTelemetryData( + eventData.operationId, + eventData.timestamp, + eventData.blockchainId, + eventData.lastEvent, + eventData.value1, + eventData.value2, + eventData.value3, + ); + }); + } + listenOnEvents(onEventReceived) { if (this.config.modules.telemetry.enabled && this.initialized) { return this.getImplementation().module.listenOnEvents( From 51ecd74014d907a40556970c0a9b9dfa8a5661a0 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Tue, 3 Dec 2024 16:19:47 +0100 Subject: [PATCH 24/29] Initialize table in sendTelemetryData --- .../implementation/quest-telemetry.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/modules/telemetry/implementation/quest-telemetry.js b/src/modules/telemetry/implementation/quest-telemetry.js index 4becd23c4..9c5fca362 100644 --- a/src/modules/telemetry/implementation/quest-telemetry.js +++ b/src/modules/telemetry/implementation/quest-telemetry.js @@ -5,7 +5,6 @@ class QuestTelemetry { this.config = config; this.logger = logger; this.sender = Sender.fromConfig(this.config.ip_endpoint); - this.table = this.sender.table('event'); } listenOnEvents(eventEmitter, onEventReceived) { @@ -22,15 +21,17 @@ class QuestTelemetry { value3 = null, ) { try { - this.table.symbol('operationId', operationId || 'NULL'); - this.table.symbol('blockchainId', blockchainId || 'NULL'); - this.table.symbol('name', name || 'NULL'); - if (value1 !== null) this.table.symbol('value1', value1); - if (value2 !== null) this.table.symbol('value2', value2); - if (value3 !== null) this.table.symbol('value3', value3); - this.table.timestampColumn('timestamp', timestamp * 1000); + const table = this.sender.table('event'); - await this.table.at(Date.now(), 'ms'); + table.symbol('operationId', operationId || 'NULL'); + table.symbol('blockchainId', blockchainId || 'NULL'); + table.symbol('name', name || 'NULL'); + if (value1 !== null) table.symbol('value1', value1); + if (value2 !== null) table.symbol('value2', value2); + if (value3 !== null) table.symbol('value3', value3); + table.timestampColumn('timestamp', timestamp * 1000); + + await table.at(Date.now(), 'ms'); await this.sender.flush(); await this.sender.close(); From 3d7d0b53cbe292c9f2fec8666f0387c668bb3291 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Thu, 5 Dec 2024 12:59:51 +0100 Subject: [PATCH 25/29] Add signaling service logic --- config/config.json | 36 ++++++++++------- src/commands/common/send-telemetry-command.js | 5 +-- .../implementation/quest-telemetry.js | 39 ++++++++++++++++--- .../telemetry/telemetry-module-manager.js | 2 +- 4 files changed, 56 insertions(+), 26 deletions(-) diff --git a/config/config.json b/config/config.json index 7d06fb706..fef2d4901 100644 --- a/config/config.json +++ b/config/config.json @@ -149,14 +149,15 @@ } }, "telemetry": { - "enabled": true, + "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false + "localEndpoint": "http::addr=localhost:10000", + "signalingServiceEndpoint": "", + "sendToSignalingService": false } } } @@ -317,14 +318,15 @@ } }, "telemetry": { - "enabled": true, + "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false + "localEndpoint": "http::addr=localhost:10000", + "signalingServiceEndpoint": "", + "sendToSignalingService": false } } } @@ -485,14 +487,15 @@ } }, "telemetry": { - "enabled": true, + "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false + "localEndpoint": "http::addr=localhost:10000", + "signalingServiceEndpoint": "", + "sendToSignalingService": false } } } @@ -652,14 +655,15 @@ } }, "telemetry": { - "enabled": true, + "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false + "localEndpoint": "http::addr=localhost:10000", + "signalingServiceEndpoint": "", + "sendToSignalingService": false } } } @@ -842,14 +846,16 @@ } }, "telemetry": { - "enabled": true, + "enabled": false, "implementation": { "quest-telemetry": { "enabled": true, "package": "./telemetry/implementation/quest-telemetry.js", "config": { - "ip_endpoint": "http::addr=localhost:10000", - "sendTelemetryData": false + "localEndpoint": "http::addr=localhost:10000", + "signalingServiceEndpoint": "", + "sendToSignalingService": false + } } } diff --git a/src/commands/common/send-telemetry-command.js b/src/commands/common/send-telemetry-command.js index ceebb8a99..3162e4eab 100644 --- a/src/commands/common/send-telemetry-command.js +++ b/src/commands/common/send-telemetry-command.js @@ -22,10 +22,7 @@ class SendTelemetryCommand extends Command { * @param command */ async execute() { - if ( - !this.config.modules.telemetry.enabled || - !this.telemetryModuleManager.getModuleConfiguration().sendTelemetryData - ) { + if (!this.config.modules.telemetry.enabled) { return Command.empty(); } diff --git a/src/modules/telemetry/implementation/quest-telemetry.js b/src/modules/telemetry/implementation/quest-telemetry.js index 9c5fca362..a71d6b953 100644 --- a/src/modules/telemetry/implementation/quest-telemetry.js +++ b/src/modules/telemetry/implementation/quest-telemetry.js @@ -4,7 +4,10 @@ class QuestTelemetry { async initialize(config, logger) { this.config = config; this.logger = logger; - this.sender = Sender.fromConfig(this.config.ip_endpoint); + this.localSender = Sender.fromConfig(this.config.localEndpoint); + if (this.config.sendToSignalingService) { + this.signalingServiceSender = Sender.fromConfig(this.config.signalingServiceEndpoint); + } } listenOnEvents(eventEmitter, onEventReceived) { @@ -21,7 +24,7 @@ class QuestTelemetry { value3 = null, ) { try { - const table = this.sender.table('event'); + const table = this.localSender.table('event'); table.symbol('operationId', operationId || 'NULL'); table.symbol('blockchainId', blockchainId || 'NULL'); @@ -32,12 +35,36 @@ class QuestTelemetry { table.timestampColumn('timestamp', timestamp * 1000); await table.at(Date.now(), 'ms'); - await this.sender.flush(); - await this.sender.close(); + await this.localSender.flush(); + await this.localSender.close(); - this.logger.info('Event telemetry successfully sent to QuestDB'); + this.logger.info('Event telemetry successfully sent to local QuestDB'); } catch (err) { - this.logger.error(`Error sending telemetry to QuestDB: ${err.message}`); + this.logger.error(`Error sending telemetry to local QuestDB: ${err.message}`); + } + + if (this.config.sendToSignalingService) { + try { + const table = this.signalingServiceSender.table('event'); + + table.symbol('operationId', operationId || 'NULL'); + table.symbol('blockchainId', blockchainId || 'NULL'); + table.symbol('name', name || 'NULL'); + if (value1 !== null) table.symbol('value1', value1); + if (value2 !== null) table.symbol('value2', value2); + if (value3 !== null) table.symbol('value3', value3); + table.timestampColumn('timestamp', timestamp * 1000); + + await table.at(Date.now(), 'ms'); + await this.signalingServiceSender.flush(); + await this.signalingServiceSender.close(); + + this.logger.info('Event telemetry successfully sent to signaling service QuestDB'); + } catch (err) { + this.logger.error( + `Error sending telemetry to signaling service QuestDB: ${err.message}`, + ); + } } } } diff --git a/src/modules/telemetry/telemetry-module-manager.js b/src/modules/telemetry/telemetry-module-manager.js index 19b8e1182..7c80fd415 100644 --- a/src/modules/telemetry/telemetry-module-manager.js +++ b/src/modules/telemetry/telemetry-module-manager.js @@ -36,7 +36,7 @@ class TelemetryModuleManager extends BaseModuleManager { } async sendTelemetryData(operationId, timestamp, blockchainId, name, value1, value2, value3) { - if (this.getImplementation().config.sendTelemetryData && this.initialized) { + if (this.config.modules.telemetry.enabled && this.initialized) { return this.getImplementation().module.sendTelemetryData( operationId, timestamp, From 236645b8b6bcd489b7e997ad40d2707929df0fa3 Mon Sep 17 00:00:00 2001 From: Zvonimir Sculac Date: Thu, 5 Dec 2024 13:02:28 +0100 Subject: [PATCH 26/29] Add send to signaling service check --- src/commands/common/send-telemetry-command.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/commands/common/send-telemetry-command.js b/src/commands/common/send-telemetry-command.js index 3162e4eab..c5af10af5 100644 --- a/src/commands/common/send-telemetry-command.js +++ b/src/commands/common/send-telemetry-command.js @@ -22,7 +22,10 @@ class SendTelemetryCommand extends Command { * @param command */ async execute() { - if (!this.config.modules.telemetry.enabled) { + if ( + !this.config.modules.telemetry.enabled || + !this.telemetryModuleManager.getModuleConfiguration().sendToSignalingService + ) { return Command.empty(); } From 8bf4566d54d9444ee915fd66ea0ef08abf11fba9 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 5 Dec 2024 13:11:30 +0100 Subject: [PATCH 27/29] fix remote vs local handling --- .../implementation/quest-telemetry.js | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/src/modules/telemetry/implementation/quest-telemetry.js b/src/modules/telemetry/implementation/quest-telemetry.js index a71d6b953..13ae80192 100644 --- a/src/modules/telemetry/implementation/quest-telemetry.js +++ b/src/modules/telemetry/implementation/quest-telemetry.js @@ -42,30 +42,6 @@ class QuestTelemetry { } catch (err) { this.logger.error(`Error sending telemetry to local QuestDB: ${err.message}`); } - - if (this.config.sendToSignalingService) { - try { - const table = this.signalingServiceSender.table('event'); - - table.symbol('operationId', operationId || 'NULL'); - table.symbol('blockchainId', blockchainId || 'NULL'); - table.symbol('name', name || 'NULL'); - if (value1 !== null) table.symbol('value1', value1); - if (value2 !== null) table.symbol('value2', value2); - if (value3 !== null) table.symbol('value3', value3); - table.timestampColumn('timestamp', timestamp * 1000); - - await table.at(Date.now(), 'ms'); - await this.signalingServiceSender.flush(); - await this.signalingServiceSender.close(); - - this.logger.info('Event telemetry successfully sent to signaling service QuestDB'); - } catch (err) { - this.logger.error( - `Error sending telemetry to signaling service QuestDB: ${err.message}`, - ); - } - } } } From e0295344e531a7613fad916ce3d2a238ec49c41e Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 5 Dec 2024 13:19:14 +0100 Subject: [PATCH 28/29] fix after conflict resolution --- ot-node.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ot-node.js b/ot-node.js index 604304a9a..bb05cad1b 100644 --- a/ot-node.js +++ b/ot-node.js @@ -54,6 +54,7 @@ class OTNode { await this.initializeRouters(); await this.startNetworkModule(); + await this.initializeBLSService(); this.resumeCommandExecutor(); this.logger.info('Node is up and running!'); } From 69ec96794a14d68b48bfd6d1cb973d5a5cfaadb1 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 5 Dec 2024 13:21:21 +0100 Subject: [PATCH 29/29] version bump --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index d4631ec92..89a8e1063 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "8.0.0-beta.6", + "version": "8.0.0-beta.7", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "8.0.0-beta.6", + "version": "8.0.0-beta.7", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 89310f0ce..698ed6501 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "8.0.0-beta.6", + "version": "8.0.0-beta.7", "description": "OTNode V8", "main": "index.js", "type": "module",