From 99a2dca124dd6e3af3311e41ca17e0fa6294d974 Mon Sep 17 00:00:00 2001 From: saki-osive Date: Sun, 2 Jun 2024 10:55:45 +0530 Subject: [PATCH 1/6] Integrate BullBoard for local development purpose --- api/package-lock.json | 151 ++++++++++++++++++++++++- api/package.json | 3 + api/src/bullboard/bullboard.service.ts | 33 ++++++ api/src/main.ts | 8 ++ 4 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 api/src/bullboard/bullboard.service.ts diff --git a/api/package-lock.json b/api/package-lock.json index 7a69e53..df5362f 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -12,6 +12,9 @@ "@apollo/server": "^4.10.4", "@aptos-labs/ts-sdk": "^1.16.0", "@aws-sdk/client-s3": "^3.583.0", + "@bull-board/api": "^5.19.2", + "@bull-board/express": "^5.19.2", + "@bull-board/nestjs": "^5.19.2", "@clickhouse/client": "^1.0.2", "@nestjs/apollo": "^12.1.0", "@nestjs/bullmq": "^10.1.1", @@ -2047,6 +2050,56 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@bull-board/api": { + "version": "5.19.2", + "resolved": "https://registry.npmjs.org/@bull-board/api/-/api-5.19.2.tgz", + "integrity": "sha512-vYXfRo7xTBeBgmC7FOKXLTHBZ/DG9ZsGIaEeE8GAkkEX1DaNKPBJOAn0Z1OmZIeLAe4esbHMFU8O1SDrmV4Hsg==", + "license": "MIT", + "dependencies": { + "redis-info": "^3.0.8" + }, + "peerDependencies": { + "@bull-board/ui": "5.19.2" + } + }, + "node_modules/@bull-board/express": { + "version": "5.19.2", + "resolved": "https://registry.npmjs.org/@bull-board/express/-/express-5.19.2.tgz", + "integrity": "sha512-CHTq7RmzE5JQAHP6ohRE6cPDqJnTnBySx5AtmbaZ0oAyhv3iMBz996cRdERrIpyfH3ymSlDrGHqJ2h0c7O629Q==", + "license": "MIT", + "dependencies": { + "@bull-board/api": "5.19.2", + "@bull-board/ui": "5.19.2", + "ejs": "^3.1.10", + "express": "^4.19.2" + } + }, + "node_modules/@bull-board/nestjs": { + "version": "5.19.2", + "resolved": "https://registry.npmjs.org/@bull-board/nestjs/-/nestjs-5.19.2.tgz", + "integrity": "sha512-qPHFtPvOrB8u6Lp1f7t0Ol1y2pexLEFH5AlsV+p8ZZh6A9h+K2T/GqMot7p5MdzjiZ2QTuR+4OutPMN2cuhmAQ==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^10.0.0" + }, + "peerDependencies": { + "@bull-board/api": "^5.19.2", + "@bull-board/express": "^5.19.2", + "@nestjs/common": "^9.0.0 || ^10.0.0", + "@nestjs/core": "^9.0.0 || ^10.0.0", + "reflect-metadata": "^0.1.13 || ^0.2.0", + "rxjs": "^7.8.1" + } + }, + "node_modules/@bull-board/ui": { + "version": "5.19.2", + "resolved": "https://registry.npmjs.org/@bull-board/ui/-/ui-5.19.2.tgz", + "integrity": "sha512-SRNK2ozJplr/5WUB2zT883d0hTb8cs61+eyrBVCJYmTKtA3uP3M8EENu/Gkv9XTZvti420qWlsZKTjOSWmm6Bg==", + "license": "MIT", + "dependencies": { + "@bull-board/api": "5.19.2" + } + }, "node_modules/@clickhouse/client": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.0.2.tgz", @@ -5562,6 +5615,12 @@ "node": ">=0.8" } }, + "node_modules/async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==", + "license": "MIT" + }, "node_modules/async-retry": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", @@ -5709,8 +5768,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/base64-js": { "version": "1.5.1", @@ -5840,7 +5898,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -6343,8 +6400,7 @@ "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "node_modules/concat-stream": { "version": "1.6.2", @@ -6829,6 +6885,21 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/ejs": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "license": "Apache-2.0", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.632", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.632.tgz", @@ -7508,6 +7579,27 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -8765,6 +8857,46 @@ "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/jake": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.1.tgz", + "integrity": "sha512-61btcOHNnLnsOdtLgA5efqQWjnSi/vow5HbI7HMdKKWqvrKR1bLK3BPlJn9gcSaP2ewuamUSMB5XEy76KUIS2w==", + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.4", + "minimatch": "^3.1.2" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jake/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/jake/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", @@ -11038,6 +11170,15 @@ "node": ">=4" } }, + "node_modules/redis-info": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redis-info/-/redis-info-3.1.0.tgz", + "integrity": "sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.11" + } + }, "node_modules/redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", diff --git a/api/package.json b/api/package.json index 1358e6c..d31d98f 100644 --- a/api/package.json +++ b/api/package.json @@ -27,6 +27,9 @@ "@apollo/server": "^4.10.4", "@aptos-labs/ts-sdk": "^1.16.0", "@aws-sdk/client-s3": "^3.583.0", + "@bull-board/api": "^5.19.2", + "@bull-board/express": "^5.19.2", + "@bull-board/nestjs": "^5.19.2", "@clickhouse/client": "^1.0.2", "@nestjs/apollo": "^12.1.0", "@nestjs/bullmq": "^10.1.1", diff --git a/api/src/bullboard/bullboard.service.ts b/api/src/bullboard/bullboard.service.ts new file mode 100644 index 0000000..8139434 --- /dev/null +++ b/api/src/bullboard/bullboard.service.ts @@ -0,0 +1,33 @@ +import { Injectable } from '@nestjs/common'; +import { Queue } from 'bullmq'; +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js'; +import { ExpressAdapter } from '@bull-board/express'; + +@Injectable() +export class BullBoardService { + + private readonly serverAdapter: ExpressAdapter; + + constructor() { + this.serverAdapter = new ExpressAdapter(); + } + + setupBullBoard(queueConfigs: { name: string; connection: any }[]) { + const queues: Queue[] = queueConfigs.map( + (config) => new Queue(config.name, { connection: config.connection }) + ); + + createBullBoard({ + queues: queues.map((queue) => new BullMQAdapter(queue)), + serverAdapter: this.serverAdapter, + }); + + this.serverAdapter.setBasePath('/ui'); + } + + getServerAdapter() { + return this.serverAdapter; + } + +} \ No newline at end of file diff --git a/api/src/main.ts b/api/src/main.ts index 07911b8..3d412ff 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -3,6 +3,7 @@ import { NestFactory } from "@nestjs/core"; import { NestExpressApplication } from "@nestjs/platform-express"; import { AppModule } from "./app/app.module.js"; import getConfig from "./config/config.js"; +import { BullBoardService } from './bullboard/bullboard.service.js'; async function bootstrap() { const config = getConfig(); @@ -15,6 +16,13 @@ async function bootstrap() { app.set("trust proxy", 1); app.enableCors(); + const isProduction = process.env.NODE_ENV === 'production'; + + if (!isProduction) { + const bullBoardService = app.get(BullBoardService); + app.use('/ui', bullBoardService.getServerAdapter().getRouter()); + } + await app.listen(config.port); } bootstrap(); From 0180b7a9d5dbf5d5384ea15730fda939f5ab1eeb Mon Sep 17 00:00:00 2001 From: celestialeo <101517557@georgebrown.ca> Date: Sun, 2 Jun 2024 11:07:27 +0530 Subject: [PATCH 2/6] Integrate BullBoard for local development purpose --- api/.env.example | 2 ++ api/src/ol/ol.module.ts | 58 +++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/api/.env.example b/api/.env.example index 41e1b99..6cd2dfe 100644 --- a/api/.env.example +++ b/api/.env.example @@ -9,6 +9,8 @@ S3_BUCKET="0lfyi-v7" S3_STORAGE_CLASS="STANDARD" S3_USE_SSL=false +NODE_ENV=local + CLICKHOUSE_HOST="127.0.0.1" CLICKHOUSE_USERNAME="default" CLICKHOUSE_PASSWORD="" diff --git a/api/src/ol/ol.module.ts b/api/src/ol/ol.module.ts index c9155ce..ffb3a6a 100644 --- a/api/src/ol/ol.module.ts +++ b/api/src/ol/ol.module.ts @@ -45,6 +45,7 @@ import { TransactionsService } from "./transactions/TransactionsService.js"; import { Transaction } from "./transactions/Transaction.js"; import { OnChainTransactionsRepository } from "./transactions/OnChainTransactionsRepository.js"; import { ExpiredTransactionsProcessor } from "./transactions/ExpiredTransactionsProcessor.js"; +import { BullBoardService } from '../bullboard/bullboard.service.js'; const roles = process.env.ROLES!.split(","); @@ -65,6 +66,16 @@ for (const role of roles) { } } +// Centralize queue definitions in an array for better reusability +const queues = [ + { name: "ol-clickhouse-ingestor", connection: redisClient}, + { name: "ol-parquet-producer", connection: redisClient }, + { name: "ol-version-batch", connection: redisClient }, + { name: "ol-version", connection: redisClient }, + { name: "expired-transactions", connection: redisClient }, +]; + + @Module({ imports: [ S3Module, @@ -74,30 +85,13 @@ for (const role of roles) { OlDbModule, WalletSubscriptionModule, - BullModule.registerQueue({ - name: "ol-clickhouse-ingestor", - connection: redisClient, - }), - - BullModule.registerQueue({ - name: "ol-parquet-producer", - connection: redisClient, - }), - - BullModule.registerQueue({ - name: "ol-version-batch", - connection: redisClient, - }), - - BullModule.registerQueue({ - name: "ol-version", - connection: redisClient, - }), - - BullModule.registerQueue({ - name: "expired-transactions", - connection: redisClient, - }), + // Register queues using a loop to simplify and maintain consistency + BullModule.registerQueue( + ...queues.map(queue => ({ + name: queue.name, + connection: queue.connection, + })) + ), ], providers: [ UserTransactionsResolver, @@ -119,6 +113,8 @@ for (const role of roles) { MovementsService, TransformerService, + BullBoardService, + // Transactions TransactionResolver, TransactionsResolver, @@ -147,6 +143,16 @@ for (const role of roles) { ...workers, ], controllers: [OlController], - exports: [OlService, TransformerService, Types.ICommunityWalletsService], + exports: [OlService, TransformerService, Types.ICommunityWalletsService, BullBoardService], }) -export class OlModule {} +export class OlModule { + isProduction = process.env.NODE_ENV === 'production'; + + constructor(private readonly bullBoardService: BullBoardService) { + if (!this.isProduction) { + // Setup BullBoard for the queues in local development environment + // Utilize the same queue array to ensure consistency + this.bullBoardService.setupBullBoard(queues); + } + } +} From dd70f57c6d26a36b3189cbfc69c1dc4ae553da21 Mon Sep 17 00:00:00 2001 From: saki-osive Date: Wed, 5 Jun 2024 08:13:10 +0530 Subject: [PATCH 3/6] Add Bullboard Service --- api/.env.example | 2 +- api/src/bullboard/bullboard.service.ts | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/api/.env.example b/api/.env.example index 6cd2dfe..84986e7 100644 --- a/api/.env.example +++ b/api/.env.example @@ -9,7 +9,7 @@ S3_BUCKET="0lfyi-v7" S3_STORAGE_CLASS="STANDARD" S3_USE_SSL=false -NODE_ENV=local +NODE_ENV= CLICKHOUSE_HOST="127.0.0.1" CLICKHOUSE_USERNAME="default" diff --git a/api/src/bullboard/bullboard.service.ts b/api/src/bullboard/bullboard.service.ts index 8139434..14099b1 100644 --- a/api/src/bullboard/bullboard.service.ts +++ b/api/src/bullboard/bullboard.service.ts @@ -3,16 +3,39 @@ import { Queue } from 'bullmq'; import { createBullBoard } from '@bull-board/api'; import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js'; import { ExpressAdapter } from '@bull-board/express'; +import { redisClient } from "../redis/redis.service.js"; @Injectable() export class BullBoardService { private readonly serverAdapter: ExpressAdapter; + private readonly queueNames: string[] = [ + "ol-clickhouse-ingestor", + "ol-parquet-producer", + "ol-version-batch", + "ol-version", + "expired-transactions", + "ol-swap", + "node-watcher", + "wallet-subscription", + "stats" + ]; constructor() { this.serverAdapter = new ExpressAdapter(); } + onModuleInit() { + if (process.env.NODE_ENV !== 'production') { + this.setupBullBoard( + this.queueNames.map(name => ({ + name, + connection: redisClient, + })) + ); + } + } + setupBullBoard(queueConfigs: { name: string; connection: any }[]) { const queues: Queue[] = queueConfigs.map( (config) => new Queue(config.name, { connection: config.connection }) From e5057293a4b9447ae9a7213d1d6c228b712183b1 Mon Sep 17 00:00:00 2001 From: celestialeo <101517557@georgebrown.ca> Date: Wed, 5 Jun 2024 08:16:21 +0530 Subject: [PATCH 4/6] Remove redundant import --- api/src/app/app.module.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/app/app.module.ts b/api/src/app/app.module.ts index 6b779d8..b9fd5ea 100644 --- a/api/src/app/app.module.ts +++ b/api/src/app/app.module.ts @@ -45,7 +45,6 @@ import { MultiSigModule } from "../multi-sig/multi-sig.module.js"; StatsModule, WalletSubscriptionModule, OlSwapModule, - NodeWatcherModule, MultiSigModule, ], controllers: [], From 1231e69eb3efb963a7a0dca6b559db9fc28a5123 Mon Sep 17 00:00:00 2001 From: celestialeo <101517557@georgebrown.ca> Date: Wed, 5 Jun 2024 08:17:28 +0530 Subject: [PATCH 5/6] Optimized queue register --- api/src/ol/ol.module.ts | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/api/src/ol/ol.module.ts b/api/src/ol/ol.module.ts index ffb3a6a..58c8a0e 100644 --- a/api/src/ol/ol.module.ts +++ b/api/src/ol/ol.module.ts @@ -67,12 +67,12 @@ for (const role of roles) { } // Centralize queue definitions in an array for better reusability -const queues = [ - { name: "ol-clickhouse-ingestor", connection: redisClient}, - { name: "ol-parquet-producer", connection: redisClient }, - { name: "ol-version-batch", connection: redisClient }, - { name: "ol-version", connection: redisClient }, - { name: "expired-transactions", connection: redisClient }, +const queueNames = [ + "ol-clickhouse-ingestor", + "ol-parquet-producer", + "ol-version-batch", + "ol-version", + "expired-transactions", ]; @@ -87,9 +87,9 @@ const queues = [ // Register queues using a loop to simplify and maintain consistency BullModule.registerQueue( - ...queues.map(queue => ({ - name: queue.name, - connection: queue.connection, + ...queueNames.map(name => ({ + name, + connection: redisClient, })) ), ], @@ -145,14 +145,4 @@ const queues = [ controllers: [OlController], exports: [OlService, TransformerService, Types.ICommunityWalletsService, BullBoardService], }) -export class OlModule { - isProduction = process.env.NODE_ENV === 'production'; - - constructor(private readonly bullBoardService: BullBoardService) { - if (!this.isProduction) { - // Setup BullBoard for the queues in local development environment - // Utilize the same queue array to ensure consistency - this.bullBoardService.setupBullBoard(queues); - } - } -} +export class OlModule {} \ No newline at end of file From c73089519e20cddee28653246f4b7e23ba79a3c8 Mon Sep 17 00:00:00 2001 From: saki-osive Date: Wed, 5 Jun 2024 18:10:08 +0530 Subject: [PATCH 6/6] Simplify environment check for BullBoardService --- api/src/main.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/main.ts b/api/src/main.ts index 3d412ff..52bae66 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -16,9 +16,7 @@ async function bootstrap() { app.set("trust proxy", 1); app.enableCors(); - const isProduction = process.env.NODE_ENV === 'production'; - - if (!isProduction) { + if (process.env.NODE_ENV !== 'production') { const bullBoardService = app.get(BullBoardService); app.use('/ui', bullBoardService.getServerAdapter().getRouter()); }