From f78ab8876f6440af00cd8029abd79a3723466f5a Mon Sep 17 00:00:00 2001 From: Sokratis Vidros Date: Fri, 8 Nov 2024 13:11:45 +0200 Subject: [PATCH 1/2] fix(dal): Reduce Mongo connections --- libs/dal/src/dal.service.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libs/dal/src/dal.service.ts b/libs/dal/src/dal.service.ts index d6e96d4ba14..2a741aca0ab 100644 --- a/libs/dal/src/dal.service.ts +++ b/libs/dal/src/dal.service.ts @@ -1,16 +1,17 @@ import mongoose, { Connection, ConnectOptions } from 'mongoose'; +export const baseConfig: ConnectOptions = { + // AUTO_CREATE_INDEXES is deprecated, use MONGO_AUTO_CREATE_INDEXES + autoIndex: process.env.AUTO_CREATE_INDEXES === 'true' || process.env.MONGO_AUTO_CREATE_INDEXES === 'true', + maxIdleTimeMS: process.env.MONGO_MAX_IDLE_TIME_IN_MS ? Number(process.env.MONGO_MAX_IDLE_TIME_IN_MS) : 1000 * 60, + maxPoolSize: process.env.MONGO_MAX_POOL_SIZE ? Number(process.env.MONGO_MAX_POOL_SIZE) : 50, + minPoolSize: process.env.MONGO_MIN_POOL_SIZE ? Number(process.env.MONGO_MIN_POOL_SIZE) : 10, +}; + export class DalService { connection: Connection; async connect(url: string, config: ConnectOptions = {}) { - const baseConfig: ConnectOptions = { - maxPoolSize: +process.env.MONGO_MAX_POOL_SIZE || 500, - minPoolSize: +process.env.MONGO_MIN_POOL_SIZE || 10, - autoIndex: process.env.AUTO_CREATE_INDEXES === 'true', - maxIdleTimeMS: 1000 * 60 * 10, - }; - const instance = await mongoose.connect(url, { ...baseConfig, ...config, From 331a08a49d28fe6cba98b6ec6927bd94e3f617ae Mon Sep 17 00:00:00 2001 From: Sokratis Vidros Date: Mon, 11 Nov 2024 13:15:40 +0200 Subject: [PATCH 2/2] fix(api): Always build SWC metadata before tet --- apps/api/package.json | 1 + apps/api/src/config/env.validators.ts | 8 ++++++-- apps/webhook/src/config/env.validators.ts | 11 +++++++++-- apps/worker/src/config/env.validators.ts | 8 ++++++-- apps/ws/src/config/env.validators.ts | 15 +++++++++++---- libs/dal/src/dal.service.ts | 2 +- 6 files changed, 34 insertions(+), 11 deletions(-) diff --git a/apps/api/package.json b/apps/api/package.json index f933fcdc655..693c459bc2d 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -20,6 +20,7 @@ "lint": "eslint src", "lint:fix": "pnpm lint -- --fix", "lint:openapi": "spectral lint http://127.0.0.1:${PORT:-3000}/openapi.yaml", + "pretest": "pnpm build:metadata", "test": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true mocha --require ts-node/register --exit 'src/**/*.spec.ts'", "test:e2e": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true mocha --require ts-node/register --exit --file e2e/setup.ts src/**/*.e2e.ts ", "test:e2e:ee": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true CI_EE_TEST=true CLERK_ENABLED=true mocha --grep @skip-in-ee --invert --require ts-node/register --exit --file e2e/setup.ts 'src/**/*.e2e{,-ee}.ts'", diff --git a/apps/api/src/config/env.validators.ts b/apps/api/src/config/env.validators.ts index c69013e775a..4f6a4b3b87f 100644 --- a/apps/api/src/config/env.validators.ts +++ b/apps/api/src/config/env.validators.ts @@ -25,9 +25,13 @@ export const envValidators = { REDIS_TLS: json({ default: undefined }), JWT_SECRET: str(), SENDGRID_API_KEY: str({ default: '' }), - MONGO_URL: str(), + /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */ + AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }), + MONGO_MAX_POOL_SIZE: num({ default: 50 }), MONGO_MIN_POOL_SIZE: num({ default: 10 }), - MONGO_MAX_POOL_SIZE: num({ default: 500 }), + MONGO_URL: str(), NOVU_API_KEY: str({ default: '' }), STORE_ENCRYPTION_KEY: str(), NEW_RELIC_APP_NAME: str({ default: '' }), diff --git a/apps/webhook/src/config/env.validators.ts b/apps/webhook/src/config/env.validators.ts index d5ae561aa5b..ad04948a6b0 100644 --- a/apps/webhook/src/config/env.validators.ts +++ b/apps/webhook/src/config/env.validators.ts @@ -1,4 +1,4 @@ -import { CleanedEnv, cleanEnv, port, str, ValidatorSpec } from 'envalid'; +import { bool, num, CleanedEnv, cleanEnv, port, str, ValidatorSpec } from 'envalid'; import { StringifyEnv } from '@novu/shared'; export function validateEnv() { @@ -8,8 +8,15 @@ export function validateEnv() { export type ValidatedEnv = StringifyEnv>; export const envValidators = { - TZ: str({ default: 'UTC' }), + /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */ + AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }), + MONGO_MAX_POOL_SIZE: num({ default: 50 }), + MONGO_MIN_POOL_SIZE: num({ default: 10 }), + MONGO_URL: str(), NODE_ENV: str({ choices: ['dev', 'test', 'production', 'ci', 'local'], default: 'local' }), PORT: port(), SENTRY_DSN: str({ default: undefined }), + TZ: str({ default: 'UTC' }), } satisfies Record>; diff --git a/apps/worker/src/config/env.validators.ts b/apps/worker/src/config/env.validators.ts index 100ff2b31ce..8c73860d22b 100644 --- a/apps/worker/src/config/env.validators.ts +++ b/apps/worker/src/config/env.validators.ts @@ -51,9 +51,13 @@ export const envValidators = { REDIS_CACHE_KEEP_ALIVE: str({ default: undefined }), REDIS_CACHE_FAMILY: str({ default: undefined }), REDIS_CACHE_KEY_PREFIX: str({ default: undefined }), - MONGO_URL: str(), + /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */ + AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }), + MONGO_MAX_POOL_SIZE: num({ default: 50 }), MONGO_MIN_POOL_SIZE: num({ default: 10 }), - MONGO_MAX_POOL_SIZE: num({ default: 500 }), + MONGO_URL: str(), SEGMENT_TOKEN: str({ default: undefined }), LAUNCH_DARKLY_SDK_KEY: str({ default: undefined }), STRIPE_API_KEY: str({ default: undefined }), diff --git a/apps/ws/src/config/env.validators.ts b/apps/ws/src/config/env.validators.ts index bfe1de5ea2b..35c9ac2fb01 100644 --- a/apps/ws/src/config/env.validators.ts +++ b/apps/ws/src/config/env.validators.ts @@ -1,4 +1,4 @@ -import { json, num, str, port, ValidatorSpec, cleanEnv, CleanedEnv } from 'envalid'; +import { json, bool, num, str, port, ValidatorSpec, cleanEnv, CleanedEnv } from 'envalid'; import { StringifyEnv } from '@novu/shared'; export function validateEnv() { @@ -8,14 +8,21 @@ export function validateEnv() { export type ValidatedEnv = StringifyEnv>; export const envValidators = { - TZ: str({ default: 'UTC' }), + JWT_SECRET: str(), + /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */ + AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_AUTO_CREATE_INDEXES: bool({ default: true }), + MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }), + MONGO_MAX_POOL_SIZE: num({ default: 50 }), + MONGO_MIN_POOL_SIZE: num({ default: 10 }), + MONGO_URL: str(), NODE_ENV: str({ choices: ['dev', 'test', 'production', 'ci', 'local'], default: 'local' }), PORT: port(), REDIS_HOST: str(), REDIS_PORT: port(), REDIS_TLS: json({ default: undefined }), - JWT_SECRET: str(), + SENTRY_DSN: str({ default: undefined }), + TZ: str({ default: 'UTC' }), WORKER_DEFAULT_CONCURRENCY: num({ default: undefined }), WORKER_DEFAULT_LOCK_DURATION: num({ default: undefined }), - SENTRY_DSN: str({ default: undefined }), } satisfies Record>; diff --git a/libs/dal/src/dal.service.ts b/libs/dal/src/dal.service.ts index 2a741aca0ab..ed1c5bfd6e0 100644 --- a/libs/dal/src/dal.service.ts +++ b/libs/dal/src/dal.service.ts @@ -3,7 +3,7 @@ import mongoose, { Connection, ConnectOptions } from 'mongoose'; export const baseConfig: ConnectOptions = { // AUTO_CREATE_INDEXES is deprecated, use MONGO_AUTO_CREATE_INDEXES autoIndex: process.env.AUTO_CREATE_INDEXES === 'true' || process.env.MONGO_AUTO_CREATE_INDEXES === 'true', - maxIdleTimeMS: process.env.MONGO_MAX_IDLE_TIME_IN_MS ? Number(process.env.MONGO_MAX_IDLE_TIME_IN_MS) : 1000 * 60, + maxIdleTimeMS: process.env.MONGO_MAX_IDLE_TIME_IN_MS ? Number(process.env.MONGO_MAX_IDLE_TIME_IN_MS) : 1000 * 30, maxPoolSize: process.env.MONGO_MAX_POOL_SIZE ? Number(process.env.MONGO_MAX_POOL_SIZE) : 50, minPoolSize: process.env.MONGO_MIN_POOL_SIZE ? Number(process.env.MONGO_MIN_POOL_SIZE) : 10, };