From c6e90f4a13db962c62da4c935762f4b01fe39959 Mon Sep 17 00:00:00 2001 From: antonleviathan Date: Sun, 12 Feb 2023 02:03:14 -0500 Subject: [PATCH 1/7] feat: draft dynamic configs --- seeds/initial-configs.js | 68 +++++++++++++++++++++++++++ src/@types/config.ts | 30 ++++++++++++ src/@types/repositories.ts | 6 +++ src/repositories/config-repository.ts | 63 +++++++++++++++++++++++++ src/utils/transform.ts | 9 ++++ 5 files changed, 176 insertions(+) create mode 100644 seeds/initial-configs.js create mode 100644 src/@types/config.ts create mode 100644 src/repositories/config-repository.ts diff --git a/seeds/initial-configs.js b/seeds/initial-configs.js new file mode 100644 index 00000000..716ac18d --- /dev/null +++ b/seeds/initial-configs.js @@ -0,0 +1,68 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +import { Category } from '../@types/category' + +const SettingsFileTypes = { + yaml: 'yaml', + json: 'json', +} +const NAMESPACE = 'c646b451-db73-47fb-9a70-ea24ce8a225a' +exports.seed = async function (knex) { + await knex('events').del() + + const { v5: uuidv5 } = require('uuid') + + const rawConfigs = getConfigs() + + const categories = Object.keys(Category) + + // TODO: Finish logic + // Do we want to flatten settings so that we can look them up by key more easily + // Or do we organize by category? If by category + const configsByCategory = categories.map(category => { + return { + id: uuidv5(event.id, NAMESPACE), + value: rawConfigs[category], + category, + } + }) + + await knex.batchInsert('configs', configsByCategory, 10) +} + +const getConfigs = () => { + const settingsFilePath = process.env.NOSTR_CONFIG_DIR ?? join(process.cwd(), '.nostr') + + const files = fs.readdirSync(settingsFilePath) + const filteredFile = files.find(fn => fn.startsWith('settings')) + + let settingsFile + if (filteredFile) { + const extension = extname(filteredFile).substring(1) + if (SettingsFileTypes[extension]) { + const extension = SettingsFileTypes[extension] + settingsFileNamePath = `${settingsFilePath}/settings.${extension}` + if (extension === SettingsFileTypes.json) { + settingsFile = loadAndParseJsonFile(settingsFileNamePath) + } else { + settingsFile = loadAndParseYamlFile(settingsFileNamePath) + } + } + } else { + settingsFile = loadAndParseYamlFile('') + } +} + +const loadAndParseJsonFile = path => { + return JSON.parse( + fs.readFileSync( + path, + { encoding: 'utf-8' } + ) + ) +} + +const loadAndParseYamlFile = path => { + const defaultSettingsFileContent = fs.readFileSync(path, { encoding: 'utf-8' }) + const defaults = yaml.load(defaultSettingsFileContent) + return defaults +} \ No newline at end of file diff --git a/src/@types/config.ts b/src/@types/config.ts new file mode 100644 index 00000000..31ef5016 --- /dev/null +++ b/src/@types/config.ts @@ -0,0 +1,30 @@ +export interface Config { + key: string, + value: object, + category: Category, + createdAt: Date, + updatedAt: Date, +} + +enum Category { + info = 'info', + payments = 'payments', + paymentsProcessors = 'paymentsProcessors', + network = 'network', + workers = 'workers', + mirroring = 'mirroring', + limits = 'limits', + // invoice + // connection + // event + // client + // message +} + +export interface DBConfig { + key: string, + value: object, + category: Category, + created_at: Date, + updated_at: Date, +} diff --git a/src/@types/repositories.ts b/src/@types/repositories.ts index 2470bff2..c9af0a99 100644 --- a/src/@types/repositories.ts +++ b/src/@types/repositories.ts @@ -2,6 +2,7 @@ import { PassThrough } from 'stream' import { DatabaseClient, EventId, Pubkey } from './base' import { DBEvent, Event } from './event' +import { Config } from './config' import { Invoice } from './invoice' import { SubscriptionFilter } from './subscription' import { User } from './user' @@ -41,3 +42,8 @@ export interface IUserRepository { upsert(user: Partial, client?: DatabaseClient): Promise getBalanceByPubkey(pubkey: Pubkey, client?: DatabaseClient): Promise } + +export interface IConfigRepository { + getConfig(key: string, client?: DatabaseClient): Promise + upsert(config: Partial, client?: DatabaseClient): Promise +} diff --git a/src/repositories/config-repository.ts b/src/repositories/config-repository.ts new file mode 100644 index 00000000..a5b46268 --- /dev/null +++ b/src/repositories/config-repository.ts @@ -0,0 +1,63 @@ +import { always, applySpec, omit, prop } from 'ramda' + +import { Config, DBConfig } from '../@types/config' +import { createLogger } from '../factories/logger-factory' +import { DatabaseClient } from '../@types/base' +import { fromDBConfig } from '../utils/transform' +import { IConfigRepository } from '../@types/repositories' + +const debug = createLogger('config-repository') + +export class ConfigRepository implements IConfigRepository { + public constructor(private readonly dbClient: DatabaseClient) { } + + public async getConfig( + key: string, + client: DatabaseClient = this.dbClient + ): Promise { + debug('find config by key: %s', key) + const [dbconfig] = await client('configs') + .where('key', key) + .select() + + if (!dbconfig) { + return + } + + return fromDBConfig(dbconfig) + } + + public async upsert( + config: Config, + client: DatabaseClient = this.dbClient, + ): Promise { + debug('upsert: %o', config) + + const date = new Date() + + const row = applySpec({ + key: prop('key'), + value: prop('value'), + category: prop('category'), + updated_at: always(date), + created_at: always(date), + })(config) + + const query = client('configs') + .insert(row) + .onConflict('key') + .merge( + omit([ + 'value', + 'category', + 'created_at', + ])(row) + ) + + return { + then: (onfulfilled: (value: number) => T1 | PromiseLike, onrejected: (reason: any) => T2 | PromiseLike) => query.then(prop('rowCount') as () => number).then(onfulfilled, onrejected), + catch: (onrejected: (reason: any) => T | PromiseLike) => query.catch(onrejected), + toString: (): string => query.toString(), + } as Promise + } +} diff --git a/src/utils/transform.ts b/src/utils/transform.ts index a5b8ccf9..6440107e 100644 --- a/src/utils/transform.ts +++ b/src/utils/transform.ts @@ -1,6 +1,7 @@ import { always, applySpec, ifElse, is, isNil, path, pipe, prop, propSatisfies } from 'ramda' import { bech32 } from 'bech32' +import { Config } from '../@types/config' import { Invoice } from '../@types/invoice' import { User } from '../@types/user' @@ -41,6 +42,14 @@ export const fromDBUser = applySpec({ updatedAt: prop('updated_at'), }) +export const fromDBConfig = applySpec({ + key: prop('key'), + value: prop('value'), + category: prop('category'), + createdAt: prop('created_at'), + updatedAt: prop('updated_at'), +}) + export const fromBech32 = (input: string) => { const { prefix, words } = bech32.decode(input) if (!input.startsWith(prefix)) { From da963fe772580fc2869ab1323c30763a97e077d4 Mon Sep 17 00:00:00 2001 From: antonleviathan Date: Wed, 15 Feb 2023 08:15:14 -0500 Subject: [PATCH 2/7] fix: don't pulverize all events in the db --- seeds/initial-configs.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/seeds/initial-configs.js b/seeds/initial-configs.js index 716ac18d..8e3122b8 100644 --- a/seeds/initial-configs.js +++ b/seeds/initial-configs.js @@ -7,8 +7,6 @@ const SettingsFileTypes = { } const NAMESPACE = 'c646b451-db73-47fb-9a70-ea24ce8a225a' exports.seed = async function (knex) { - await knex('events').del() - const { v5: uuidv5 } = require('uuid') const rawConfigs = getConfigs() From e5c070b66b732755c04a079b9fad37f7657c1e18 Mon Sep 17 00:00:00 2001 From: antonleviathan Date: Sat, 18 Feb 2023 22:29:31 -0500 Subject: [PATCH 3/7] refactor configs to be loaded using callback in a singleton --- resources/default-settings.yaml | 2 +- seeds/configs.json | 368 ++++++++++++++++++ seeds/initial-configs.js | 68 +++- src/@types/repositories.ts | 9 +- src/@types/{config.ts => setting.ts} | 4 +- src/app/app.ts | 2 +- src/app/worker.ts | 3 - src/index.ts | 47 ++- ...g-repository.ts => settings-repository.ts} | 44 ++- src/utils/event.ts | 2 +- src/utils/settings.ts | 153 +++----- src/utils/transform.ts | 4 +- test/integration/features/shared.ts | 2 +- 13 files changed, 535 insertions(+), 173 deletions(-) create mode 100644 seeds/configs.json rename src/@types/{config.ts => setting.ts} (89%) rename src/repositories/{config-repository.ts => settings-repository.ts} (57%) diff --git a/resources/default-settings.yaml b/resources/default-settings.yaml index 62bd4c4e..4351fd8d 100755 --- a/resources/default-settings.yaml +++ b/resources/default-settings.yaml @@ -10,7 +10,7 @@ payments: feeSchedules: admission: - enabled: false - descripton: Admission fee charged per public key in msats (1000 msats = 1 satoshi) + description: Admission fee charged per public key in msats (1000 msats = 1 satoshi) amount: 1000000 whitelists: pubkeys: diff --git a/seeds/configs.json b/seeds/configs.json new file mode 100644 index 00000000..f457d1d1 --- /dev/null +++ b/seeds/configs.json @@ -0,0 +1,368 @@ +[ + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "relay_url", + "value": "wss://nostream.your-domain.com", + "category": "info" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "name", + "value": "nostream.your-domain.com", + "category": "info" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "description", + "value": "A nostr relay written in Typescript.", + "category": "info" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "pubkey", + "value": "replace-with-your-pubkey-in-hex", + "category": "info" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "contact", + "value": "operator@your-domain.com", + "category": "info" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "enabled", + "value": false, + "category": "authentication" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "enabled", + "value": false, + "category": "payments" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "processor", + "value": "zebedee", + "category": "payments" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "feeSchedules", + "value": { + "admission": [ + { + "enabled": false, + "description": "Admission fee charged per public key in msats (1000 msats = 1 satoshi)", + "amount": 1000000, + "whitelists": { + "pubkeys": [ + "replace-with-your-pubkey-in-hex" + ] + } + } + ], + "publication": [ + { + "enabled": false, + "description": "Publication fee charged per event in msats (1000 msats = 1 satoshi)", + "amount": 10, + "whitelists": { + "pubkeys": [ + "replace-with-your-pubkey-in-hex" + ] + } + } + ] + }, + "category": "payments" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "zebedee", + "value": { + "baseURL": "https://api.zebedee.io/", + "callbackBaseURL": "https://nostream.your-domain.com/callbacks/zebedee", + "ipWhitelist": [ + "3.225.112.64", + "::ffff:3.225.112.64" + ] + }, + "category": "paymentsProcessors" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "maxPayloadSize", + "value": 131072, + "category": "network" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "remoteIpHeader", + "value": "x-forwarded-for", + "category": "network" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "idleTimeout", + "value": 60, + "category": "network" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "count", + "value": 0, + "category": "workers" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "static", + "value": [], + "category": "mirroring" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "invoice", + "value": { + "rateLimits": [ + { + "period": 60000, + "rate": 3 + }, + { + "period": 3600000, + "rate": 10 + }, + { + "period": 86400000, + "rate": 20 + } + ], + "ipWhitelist": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + }, + "category": "limits" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "connection", + "value": { + "rateLimits": [ + { + "period": 1000, + "rate": 6 + }, + { + "period": 60000, + "rate": 30 + }, + { + "period": 3600000, + "rate": 300 + }, + { + "period": 86400000, + "rate": 1440 + } + ], + "ipWhitelist": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + }, + "category": "limits" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "event", + "value": { + "eventId": { + "minLeadingZeroBits": 0 + }, + "kind": { + "whitelist": [], + "blacklist": [] + }, + "pubkey": { + "minBalance": 0, + "minLeadingZeroBits": 0, + "whitelist": [], + "blacklist": [] + }, + "createdAt": { + "maxPositiveDelta": 900, + "maxNegativeDelta": 0 + }, + "content": [ + { + "description": "64 KB for event kind ranges 0-10 and 40-49", + "kinds": [ + [ + 0, + 10 + ], + [ + 40, + 49 + ] + ], + "maxLength": 65536 + }, + { + "description": "96 KB for event kind ranges 11-39 and 50-max", + "kinds": [ + [ + 11, + 39 + ], + [ + 50, + 9007199254740991 + ] + ], + "maxLength": 98304 + } + ], + "rateLimits": [ + { + "description": "6 events/min for event kinds 0, 3, 40 and 41", + "kinds": [ + 0, + 3, + 40, + 41 + ], + "period": 60000, + "rate": 6 + }, + { + "description": "12 events/min for event kinds 1, 2, 4 and 42", + "kinds": [ + 1, + 2, + 4, + 42 + ], + "period": 60000, + "rate": 12 + }, + { + "description": "360 events/hour for event kinds 1, 2, 4 and 42", + "kinds": [ + 1, + 2, + 4, + 42 + ], + "period": 3600000, + "rate": 360 + }, + { + "description": "30 events/min for event kind ranges 5-7 and 43-49", + "kinds": [ + [ + 5, + 7 + ], + [ + 43, + 49 + ] + ], + "period": 60000, + "rate": 30 + }, + { + "description": "24 events/min for replaceable events and parameterized replaceable events", + "kinds": [ + [ + 10000, + 19999 + ], + [ + 30000, + 39999 + ] + ], + "period": 60000, + "rate": 24 + }, + { + "description": "60 events/min for ephemeral events", + "kinds": [ + [ + 20000, + 29999 + ] + ], + "period": 60000, + "rate": 60 + }, + { + "description": "720 events/hour for all events", + "period": 3600000, + "rate": 720 + }, + { + "description": "2880 events/day for all events", + "period": 86400000, + "rate": 2880 + } + ], + "whitelists": { + "pubkeys": [], + "ipAddresses": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + } + }, + "category": "limits" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "client", + "value": { + "subscription": { + "maxSubscriptions": 10, + "maxFilters": 10 + } + }, + "category": "limits" + }, + { + "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", + "key": "message", + "value": { + "rateLimits": [ + { + "description": "120 raw messages/min", + "period": 60000, + "rate": 120 + }, + { + "description": "3600 raw messages/hour", + "period": 3600000, + "rate": 3600 + }, + { + "description": "86400 raw messages/day", + "period": 86400000, + "rate": 86400 + } + ], + "ipWhitelist": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + }, + "category": "limits" + } +] \ No newline at end of file diff --git a/seeds/initial-configs.js b/seeds/initial-configs.js index 8e3122b8..36bcf79a 100644 --- a/seeds/initial-configs.js +++ b/seeds/initial-configs.js @@ -1,43 +1,46 @@ /* eslint-disable @typescript-eslint/no-var-requires */ -import { Category } from '../@types/category' +const { extname, join } = require('path') +const fs = require('fs') +const yaml = require('js-yaml') +const { v5: uuidv5 } = require('uuid') const SettingsFileTypes = { yaml: 'yaml', json: 'json', } + const NAMESPACE = 'c646b451-db73-47fb-9a70-ea24ce8a225a' + exports.seed = async function (knex) { - const { v5: uuidv5 } = require('uuid') + const settingsFilePath = `${process.cwd()}/seeds/configs.json` + const defaultConfigs = fs.readFileSync(settingsFilePath) + // await knew.batchInsert('configs', defaultConfigs, 10) const rawConfigs = getConfigs() - const categories = Object.keys(Category) - - // TODO: Finish logic - // Do we want to flatten settings so that we can look them up by key more easily - // Or do we organize by category? If by category - const configsByCategory = categories.map(category => { - return { - id: uuidv5(event.id, NAMESPACE), - value: rawConfigs[category], - category, - } - }) + const parsedConfigs = parseAll(rawConfigs) - await knex.batchInsert('configs', configsByCategory, 10) + if (parsedConfigs) { + // await knex.batchInsert('configs', configsByCategory, 10) + } } const getConfigs = () => { const settingsFilePath = process.env.NOSTR_CONFIG_DIR ?? join(process.cwd(), '.nostr') const files = fs.readdirSync(settingsFilePath) + const settingsFilesTotal = files.filter(file => file.match(/settings/)) + + if (settingsFilesTotal.length > 1) { + throw new Error('There are more than 1 settings file, please delete all files that contain the word settings in their name, and restart the relay') + } + const filteredFile = files.find(fn => fn.startsWith('settings')) let settingsFile if (filteredFile) { const extension = extname(filteredFile).substring(1) if (SettingsFileTypes[extension]) { - const extension = SettingsFileTypes[extension] settingsFileNamePath = `${settingsFilePath}/settings.${extension}` if (extension === SettingsFileTypes.json) { settingsFile = loadAndParseJsonFile(settingsFileNamePath) @@ -45,9 +48,9 @@ const getConfigs = () => { settingsFile = loadAndParseYamlFile(settingsFileNamePath) } } - } else { - settingsFile = loadAndParseYamlFile('') } + + return settingsFile } const loadAndParseJsonFile = path => { @@ -63,4 +66,31 @@ const loadAndParseYamlFile = path => { const defaultSettingsFileContent = fs.readFileSync(path, { encoding: 'utf-8' }) const defaults = yaml.load(defaultSettingsFileContent) return defaults -} \ No newline at end of file +} + +const parseAll = (jsonConfigs) => { + if (!jsonConfigs) return + + const keys = Object.keys(jsonConfigs) + + const configs = keys.map(key => { + return parseOneLevelDeepConfigs(jsonConfigs[key], key) + }) + + return configs.flat() +} + +const parseOneLevelDeepConfigs = (configs, category) => { + const keys = Object.keys(configs) + console.log(keys) + const flattenedConfigs = Object.keys(configs).map(key => { + return { + id: uuidv5('id', NAMESPACE), + key, + value: configs[key], + category + } + }) + + return flattenedConfigs +} diff --git a/src/@types/repositories.ts b/src/@types/repositories.ts index c9af0a99..bd2fc935 100644 --- a/src/@types/repositories.ts +++ b/src/@types/repositories.ts @@ -2,7 +2,7 @@ import { PassThrough } from 'stream' import { DatabaseClient, EventId, Pubkey } from './base' import { DBEvent, Event } from './event' -import { Config } from './config' +import { Setting } from './setting' import { Invoice } from './invoice' import { SubscriptionFilter } from './subscription' import { User } from './user' @@ -43,7 +43,8 @@ export interface IUserRepository { getBalanceByPubkey(pubkey: Pubkey, client?: DatabaseClient): Promise } -export interface IConfigRepository { - getConfig(key: string, client?: DatabaseClient): Promise - upsert(config: Partial, client?: DatabaseClient): Promise +export interface ISettingRepository { + getSetting(category: string, key: string, client?: DatabaseClient): Promise + getSettings(): Promise + upsertSetting(config: Partial, client?: DatabaseClient): Promise } diff --git a/src/@types/config.ts b/src/@types/setting.ts similarity index 89% rename from src/@types/config.ts rename to src/@types/setting.ts index 31ef5016..2a93499e 100644 --- a/src/@types/config.ts +++ b/src/@types/setting.ts @@ -1,4 +1,4 @@ -export interface Config { +export interface Setting { key: string, value: object, category: Category, @@ -21,7 +21,7 @@ enum Category { // message } -export interface DBConfig { +export interface DBSetting { key: string, value: object, category: Category, diff --git a/src/app/app.ts b/src/app/app.ts index 7e02cbbc..fb83a42c 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -38,7 +38,7 @@ export class App implements IRunnable { public run(): void { const settings = this.settings() - this.watchers = SettingsStatic.watchSettings() + // this.watchers = SettingsStatic.watchSettings() console.log(` ███▄ █ ▒█████ ██████ ▄▄▄█████▓ ██▀███ ▓█████ ▄▄▄ ███▄ ▄███▓ ██ ▀█ █ ▒██▒ ██▒▒██ ▒ ▓ ██▒ ▓▒▓██ ▒ ██▒▓█ ▀▒████▄ ▓██▒▀█▀ ██▒ diff --git a/src/app/worker.ts b/src/app/worker.ts index b5cce811..b6a1adad 100644 --- a/src/app/worker.ts +++ b/src/app/worker.ts @@ -3,7 +3,6 @@ import { IWebSocketServerAdapter } from '../@types/adapters' import { createLogger } from '../factories/logger-factory' import { FSWatcher } from 'fs' -import { SettingsStatic } from '../utils/settings' const debug = createLogger('app-worker') export class AppWorker implements IRunnable { @@ -23,8 +22,6 @@ export class AppWorker implements IRunnable { } public run(): void { - this.watchers = SettingsStatic.watchSettings() - const port = process.env.PORT || process.env.RELAY_PORT || 8008 this.adapter.listen(typeof port === 'number' ? port : Number(port)) } diff --git a/src/index.ts b/src/index.ts index 7d8ef98b..4fd87533 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,24 +6,39 @@ import { appFactory } from './factories/app-factory' import { maintenanceWorkerFactory } from './factories/maintenance-worker-factory' import { staticMirroringWorkerFactory } from './factories/static-mirroring.worker-factory' import { workerFactory } from './factories/worker-factory' +import { getMasterDbClient } from './database/client' +import { SettingsStatic } from './utils/settings' -export const getRunner = () => { - if (cluster.isPrimary) { - return appFactory() - } else { - switch (process.env.WORKER_TYPE) { - case 'worker': - return workerFactory() - case 'maintenance': - return maintenanceWorkerFactory() - case 'static-mirroring': - return staticMirroringWorkerFactory() - default: - throw new Error(`Unknown worker: ${process.env.WORKER_TYPE}`) - } - } +export const getRunner = (): any => { + const dbClient = getMasterDbClient() + const initializeSettings = new SettingsStatic(dbClient).init() + console.log('here1i') + + initializeSettings + .then(() => { + if (cluster.isPrimary) { + appFactory().run() + } else { + switch (process.env.WORKER_TYPE) { + case 'worker': + workerFactory().run() + return + case 'maintenance': + maintenanceWorkerFactory().run() + return + case 'static-mirroring': + staticMirroringWorkerFactory().run() + return + default: + throw new Error(`Unknown worker: ${process.env.WORKER_TYPE}`) + } + } + }) + .catch(error => { + throw new Error('Failed to load settings', error) + }) } if (require.main === module) { - getRunner().run() + getRunner() } diff --git a/src/repositories/config-repository.ts b/src/repositories/settings-repository.ts similarity index 57% rename from src/repositories/config-repository.ts rename to src/repositories/settings-repository.ts index a5b46268..5f0e18bf 100644 --- a/src/repositories/config-repository.ts +++ b/src/repositories/settings-repository.ts @@ -1,41 +1,57 @@ import { always, applySpec, omit, prop } from 'ramda' -import { Config, DBConfig } from '../@types/config' +import { Setting, DBSetting } from '../@types/setting' import { createLogger } from '../factories/logger-factory' import { DatabaseClient } from '../@types/base' -import { fromDBConfig } from '../utils/transform' -import { IConfigRepository } from '../@types/repositories' +import { fromDBSetting } from '../utils/transform' +import { ISettingRepository } from '../@types/repositories' const debug = createLogger('config-repository') -export class ConfigRepository implements IConfigRepository { +export class SettingRepository implements ISettingRepository { public constructor(private readonly dbClient: DatabaseClient) { } - public async getConfig( + public async getSetting( + category: string, key: string, client: DatabaseClient = this.dbClient - ): Promise { - debug('find config by key: %s', key) - const [dbconfig] = await client('configs') + ): Promise { + debug('find config by key: %s and category %s', category, key); + const [dbsetting] = await client('configs') .where('key', key) + .where('category', category) .select() - if (!dbconfig) { + if (!dbsetting) { return } - return fromDBConfig(dbconfig) + return fromDBSetting(dbsetting) } - public async upsert( - config: Config, + public async getSettings( + client: DatabaseClient = this.dbClient + ): Promise { + debug('get all configs'); + const settings = await client('configs') + .select() + + if (!settings) { + return + } + + return settings + } + + public async upsertSetting( + config: Setting, client: DatabaseClient = this.dbClient, ): Promise { debug('upsert: %o', config) const date = new Date() - const row = applySpec({ + const row = applySpec({ key: prop('key'), value: prop('value'), category: prop('category'), @@ -43,7 +59,7 @@ export class ConfigRepository implements IConfigRepository { created_at: always(date), })(config) - const query = client('configs') + const query = client('configs') .insert(row) .onConflict('key') .merge( diff --git a/src/utils/event.ts b/src/utils/event.ts index 7edd0263..133a28c4 100644 --- a/src/utils/event.ts +++ b/src/utils/event.ts @@ -289,7 +289,7 @@ export const getEventExpiration = (event: Event): number | undefined => { const expirationTime = Number(rawExpirationTime) if ((Number.isSafeInteger(expirationTime) && Math.log10(expirationTime))) { return expirationTime - } + } } export const getEventProofOfWork = (eventId: EventId): number => { diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 55092c33..5e70f286 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -1,138 +1,73 @@ -import fs from 'fs' -import yaml from 'js-yaml' - -import { extname, join } from 'path' -import { mergeDeepRight } from 'ramda' - import { createLogger } from '../factories/logger-factory' import { Settings } from '../@types/settings' +import { SettingRepository } from '../repositories/settings-repository' +import { Setting } from '../@types/setting' +import { DatabaseClient } from '../@types/base' const debug = createLogger('settings') -export enum SettingsFileTypes { - yaml = 'yaml', - json = 'json', -} - export class SettingsStatic { + private static _instance: SettingsStatic + private static dbClient: DatabaseClient static _settings: Settings | undefined + static settingsRepository: SettingRepository | undefined - public static getSettingsFileBasePath(): string { - return process.env.NOSTR_CONFIG_DIR ?? join(process.cwd(), '.nostr') - } + constructor(dbClient: DatabaseClient) { + SettingsStatic.dbClient = dbClient + SettingsStatic.settingsRepository = new SettingRepository(dbClient) + if (SettingsStatic._instance) + throw new Error("Use Singleton.instance instead of new.") - public static getDefaultSettingsFilePath(): string { - return join(process.cwd(), 'resources', 'default-settings.yaml') + SettingsStatic._instance = this; } - public static loadAndParseYamlFile(path: string): Settings { - const defaultSettingsFileContent = fs.readFileSync(path, { encoding: 'utf-8' }) - const defaults = yaml.load(defaultSettingsFileContent) as Settings - return defaults + public init() { + debug('SettingsStatic.init()') + return new Promise((resolve, reject) => { + const settingsPromise = SettingsStatic.loadSettingsFromDb(SettingsStatic.constructSettingsJsonBlob) + if (settingsPromise) { + resolve('success') + } + reject('Failed to initialize settings') + }) } - public static loadAndParseJsonFile(path: string) { - return JSON.parse( - fs.readFileSync( - path, - { encoding: 'utf-8' } - ) - ) + static get instance() { + return SettingsStatic._instance ?? (SettingsStatic._instance = new SettingsStatic(this.dbClient)); } - public static settingsFileType(path: string): SettingsFileTypes | undefined { - const files: string[] = fs.readdirSync(path) - const filteredFile = files.find(fn => fn.startsWith('settings')) - if (filteredFile) { - const extension = extname(filteredFile).substring(1) - if (SettingsFileTypes[extension]) { - return SettingsFileTypes[extension] - } - } + private static loadSettingsFromDb(callback) { + debug('SettingsStatic.loadSettingsFromDb()') + const promise = SettingsStatic.settingsRepository.getSettings() - return SettingsFileTypes.yaml + return promise.then(rawSettingsFromDb => { + const settingsJsonBlob = callback(rawSettingsFromDb); + this._settings = settingsJsonBlob + }); } - public static loadSettings(path: string, fileType: SettingsFileTypes) { - debug('loading settings from %s', path) - - switch (fileType) { - case SettingsFileTypes.json: { - console.warn('settings.json is deprecated, please use a yaml file based on resources/default-settings.yaml') - return SettingsStatic.loadAndParseJsonFile(path) - } - case SettingsFileTypes.yaml: { - return SettingsStatic.loadAndParseYamlFile(path) - } - default: { - throw new Error('settings file was missing or did not contain .yaml or .json extensions.') - } - } - } public static createSettings(): Settings { - if (SettingsStatic._settings) { - return SettingsStatic._settings - } - debug('creating settings') - - const basePath = SettingsStatic.getSettingsFileBasePath() - if (!fs.existsSync(basePath)) { - fs.mkdirSync(basePath) - } - const defaultsFilePath = SettingsStatic.getDefaultSettingsFilePath() - const fileType = SettingsStatic.settingsFileType(basePath) - const settingsFilePath = join(basePath, `settings.${fileType}`) - - const defaults = SettingsStatic.loadSettings(defaultsFilePath, SettingsFileTypes.yaml) - - try { - if (fileType) { - SettingsStatic._settings = mergeDeepRight( - defaults, - SettingsStatic.loadSettings(settingsFilePath, fileType) - ) - } else { - SettingsStatic.saveSettings(basePath, defaults) - SettingsStatic._settings = mergeDeepRight({}, defaults) - } - - if (typeof SettingsStatic._settings === 'undefined') { - throw new Error('Unable to set settings') - } + return this._settings + } - return SettingsStatic._settings - } catch (error) { - debug('error reading config file at %s: %o', settingsFilePath, error) + public static async updateSetting(config: Setting) { + await SettingsStatic.settingsRepository.upsertSetting(config) - return defaults - } + this.updateSingletonSettings(config) } - public static saveSettings(path: string, settings: Settings) { - debug('saving settings to %s: %o', path, settings) - return fs.writeFileSync( - join(path, 'settings.yaml'), - yaml.dump(settings), - { encoding: 'utf-8' }, - ) + private static updateSingletonSettings(setting) { + const updateSettings = this._settings + updateSettings[setting.category][setting.key] = setting.value } - public static watchSettings() { - const basePath = SettingsStatic.getSettingsFileBasePath() - const defaultsFilePath = SettingsStatic.getDefaultSettingsFilePath() - const fileType = SettingsStatic.settingsFileType(basePath) - const settingsFilePath = join(basePath, `settings.${fileType}`) - - const reload = () => { - console.log('reloading settings') - SettingsStatic._settings = undefined - SettingsStatic.createSettings() - } + private static constructSettingsJsonBlob(rawSettingsFromDb): any { + const settings = {} + rawSettingsFromDb.map(setting => { + settings[setting.category][setting.key] = setting.value + }) - return [ - fs.watch(defaultsFilePath, 'utf8', reload), - fs.watch(settingsFilePath, 'utf8', reload), - ] + return settings } } diff --git a/src/utils/transform.ts b/src/utils/transform.ts index 6440107e..01ebd102 100644 --- a/src/utils/transform.ts +++ b/src/utils/transform.ts @@ -1,9 +1,9 @@ import { always, applySpec, ifElse, is, isNil, path, pipe, prop, propSatisfies } from 'ramda' import { bech32 } from 'bech32' -import { Config } from '../@types/config' import { Invoice } from '../@types/invoice' import { User } from '../@types/user' +import { Setting } from '../@types/setting' export const toJSON = (input: any) => JSON.stringify(input) @@ -42,7 +42,7 @@ export const fromDBUser = applySpec({ updatedAt: prop('updated_at'), }) -export const fromDBConfig = applySpec({ +export const fromDBSetting = applySpec({ key: prop('key'), value: prop('value'), category: prop('category'), diff --git a/test/integration/features/shared.ts b/test/integration/features/shared.ts index 38aaa854..9568feb7 100644 --- a/test/integration/features/shared.ts +++ b/test/integration/features/shared.ts @@ -39,7 +39,7 @@ BeforeAll({ timeout: 1000 }, async function () { dbClient = getMasterDbClient() rrDbClient = getReadReplicaDbClient() await dbClient.raw('SELECT 1=1') - Sinon.stub(SettingsStatic, 'watchSettings') + // Sinon.stub(SettingsStatic, 'watchSettings') const settings = SettingsStatic.createSettings() SettingsStatic._settings = pipe( From 34f4c93d9d10f64c5f2cfcf635001069a25963c3 Mon Sep 17 00:00:00 2001 From: antonleviathan Date: Sun, 12 Mar 2023 16:12:38 -0400 Subject: [PATCH 4/7] refactor: linting and pr feedback --- seeds/initial-configs.js | 2 +- src/@types/repositories.ts | 2 +- src/app/app.ts | 2 -- src/index.ts | 4 ++-- src/repositories/settings-repository.ts | 6 +++--- src/utils/settings.ts | 19 ++++++++++--------- src/utils/transform.ts | 2 +- test/integration/features/shared.ts | 2 -- 8 files changed, 18 insertions(+), 21 deletions(-) diff --git a/seeds/initial-configs.js b/seeds/initial-configs.js index 36bcf79a..b1ed96b2 100644 --- a/seeds/initial-configs.js +++ b/seeds/initial-configs.js @@ -85,7 +85,7 @@ const parseOneLevelDeepConfigs = (configs, category) => { console.log(keys) const flattenedConfigs = Object.keys(configs).map(key => { return { - id: uuidv5('id', NAMESPACE), + id: uuidv5('key', NAMESPACE), key, value: configs[key], category diff --git a/src/@types/repositories.ts b/src/@types/repositories.ts index bd2fc935..efa5fc61 100644 --- a/src/@types/repositories.ts +++ b/src/@types/repositories.ts @@ -2,8 +2,8 @@ import { PassThrough } from 'stream' import { DatabaseClient, EventId, Pubkey } from './base' import { DBEvent, Event } from './event' -import { Setting } from './setting' import { Invoice } from './invoice' +import { Setting } from './setting' import { SubscriptionFilter } from './subscription' import { User } from './user' diff --git a/src/app/app.ts b/src/app/app.ts index fb83a42c..65a7ea9b 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -9,7 +9,6 @@ import { IRunnable } from '../@types/base' import packageJson from '../../package.json' import { Serializable } from 'child_process' import { Settings } from '../@types/settings' -import { SettingsStatic } from '../utils/settings' const debug = createLogger('app-primary') @@ -38,7 +37,6 @@ export class App implements IRunnable { public run(): void { const settings = this.settings() - // this.watchers = SettingsStatic.watchSettings() console.log(` ███▄ █ ▒█████ ██████ ▄▄▄█████▓ ██▀███ ▓█████ ▄▄▄ ███▄ ▄███▓ ██ ▀█ █ ▒██▒ ██▒▒██ ▒ ▓ ██▒ ▓▒▓██ ▒ ██▒▓█ ▀▒████▄ ▓██▒▀█▀ ██▒ diff --git a/src/index.ts b/src/index.ts index 4fd87533..33c71e93 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,11 +3,11 @@ import dotenv from 'dotenv' dotenv.config() import { appFactory } from './factories/app-factory' +import { getMasterDbClient } from './database/client' import { maintenanceWorkerFactory } from './factories/maintenance-worker-factory' +import { SettingsStatic } from './utils/settings' import { staticMirroringWorkerFactory } from './factories/static-mirroring.worker-factory' import { workerFactory } from './factories/worker-factory' -import { getMasterDbClient } from './database/client' -import { SettingsStatic } from './utils/settings' export const getRunner = (): any => { const dbClient = getMasterDbClient() diff --git a/src/repositories/settings-repository.ts b/src/repositories/settings-repository.ts index 5f0e18bf..a398a450 100644 --- a/src/repositories/settings-repository.ts +++ b/src/repositories/settings-repository.ts @@ -1,6 +1,6 @@ import { always, applySpec, omit, prop } from 'ramda' -import { Setting, DBSetting } from '../@types/setting' +import { DBSetting, Setting } from '../@types/setting' import { createLogger } from '../factories/logger-factory' import { DatabaseClient } from '../@types/base' import { fromDBSetting } from '../utils/transform' @@ -16,7 +16,7 @@ export class SettingRepository implements ISettingRepository { key: string, client: DatabaseClient = this.dbClient ): Promise { - debug('find config by key: %s and category %s', category, key); + debug('find config by key: %s and category %s', category, key) const [dbsetting] = await client('configs') .where('key', key) .where('category', category) @@ -32,7 +32,7 @@ export class SettingRepository implements ISettingRepository { public async getSettings( client: DatabaseClient = this.dbClient ): Promise { - debug('get all configs'); + debug('get all configs') const settings = await client('configs') .select() diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 5e70f286..b1344c01 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -1,8 +1,8 @@ import { createLogger } from '../factories/logger-factory' -import { Settings } from '../@types/settings' -import { SettingRepository } from '../repositories/settings-repository' -import { Setting } from '../@types/setting' import { DatabaseClient } from '../@types/base' +import { Setting } from '../@types/setting' +import { SettingRepository } from '../repositories/settings-repository' +import { Settings } from '../@types/settings' const debug = createLogger('settings') @@ -15,10 +15,11 @@ export class SettingsStatic { constructor(dbClient: DatabaseClient) { SettingsStatic.dbClient = dbClient SettingsStatic.settingsRepository = new SettingRepository(dbClient) - if (SettingsStatic._instance) - throw new Error("Use Singleton.instance instead of new.") + if (SettingsStatic._instance) { + throw new Error('Use Singleton.instance instead of new.') + } - SettingsStatic._instance = this; + SettingsStatic._instance = this } public init() { @@ -33,7 +34,7 @@ export class SettingsStatic { } static get instance() { - return SettingsStatic._instance ?? (SettingsStatic._instance = new SettingsStatic(this.dbClient)); + return SettingsStatic._instance ?? (SettingsStatic._instance = new SettingsStatic(this.dbClient)) } private static loadSettingsFromDb(callback) { @@ -41,9 +42,9 @@ export class SettingsStatic { const promise = SettingsStatic.settingsRepository.getSettings() return promise.then(rawSettingsFromDb => { - const settingsJsonBlob = callback(rawSettingsFromDb); + const settingsJsonBlob = callback(rawSettingsFromDb) this._settings = settingsJsonBlob - }); + }) } diff --git a/src/utils/transform.ts b/src/utils/transform.ts index 01ebd102..459940ff 100644 --- a/src/utils/transform.ts +++ b/src/utils/transform.ts @@ -2,8 +2,8 @@ import { always, applySpec, ifElse, is, isNil, path, pipe, prop, propSatisfies } import { bech32 } from 'bech32' import { Invoice } from '../@types/invoice' -import { User } from '../@types/user' import { Setting } from '../@types/setting' +import { User } from '../@types/user' export const toJSON = (input: any) => JSON.stringify(input) diff --git a/test/integration/features/shared.ts b/test/integration/features/shared.ts index 9568feb7..3965514d 100644 --- a/test/integration/features/shared.ts +++ b/test/integration/features/shared.ts @@ -11,7 +11,6 @@ import { import { assocPath, pipe } from 'ramda' import { fromEvent, map, Observable, ReplaySubject, Subject, takeUntil } from 'rxjs' import WebSocket, { MessageEvent } from 'ws' -import Sinon from 'sinon' import { connect, createIdentity, createSubscription, sendEvent } from './helpers' import { getMasterDbClient, getReadReplicaDbClient } from '../../../src/database/client' @@ -39,7 +38,6 @@ BeforeAll({ timeout: 1000 }, async function () { dbClient = getMasterDbClient() rrDbClient = getReadReplicaDbClient() await dbClient.raw('SELECT 1=1') - // Sinon.stub(SettingsStatic, 'watchSettings') const settings = SettingsStatic.createSettings() SettingsStatic._settings = pipe( From 93e561f00eca156cf61ef99b4f64ec466ee63c7f Mon Sep 17 00:00:00 2001 From: antonleviathan Date: Sun, 12 Mar 2023 16:42:55 -0400 Subject: [PATCH 5/7] fix: ids issue --- seeds/configs.json | 20 -------------------- seeds/initial-configs.js | 23 +++++++++++++++++------ 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/seeds/configs.json b/seeds/configs.json index f457d1d1..04d0ca6c 100644 --- a/seeds/configs.json +++ b/seeds/configs.json @@ -1,54 +1,45 @@ [ { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "relay_url", "value": "wss://nostream.your-domain.com", "category": "info" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "name", "value": "nostream.your-domain.com", "category": "info" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "description", "value": "A nostr relay written in Typescript.", "category": "info" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "pubkey", "value": "replace-with-your-pubkey-in-hex", "category": "info" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "contact", "value": "operator@your-domain.com", "category": "info" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "enabled", "value": false, "category": "authentication" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "enabled", "value": false, "category": "payments" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "processor", "value": "zebedee", "category": "payments" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "feeSchedules", "value": { "admission": [ @@ -79,7 +70,6 @@ "category": "payments" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "zebedee", "value": { "baseURL": "https://api.zebedee.io/", @@ -92,37 +82,31 @@ "category": "paymentsProcessors" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "maxPayloadSize", "value": 131072, "category": "network" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "remoteIpHeader", "value": "x-forwarded-for", "category": "network" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "idleTimeout", "value": 60, "category": "network" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "count", "value": 0, "category": "workers" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "static", "value": [], "category": "mirroring" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "invoice", "value": { "rateLimits": [ @@ -148,7 +132,6 @@ "category": "limits" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "connection", "value": { "rateLimits": [ @@ -178,7 +161,6 @@ "category": "limits" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "event", "value": { "eventId": { @@ -326,7 +308,6 @@ "category": "limits" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "client", "value": { "subscription": { @@ -337,7 +318,6 @@ "category": "limits" }, { - "id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13", "key": "message", "value": { "rateLimits": [ diff --git a/seeds/initial-configs.js b/seeds/initial-configs.js index b1ed96b2..5aef721f 100644 --- a/seeds/initial-configs.js +++ b/seeds/initial-configs.js @@ -3,6 +3,7 @@ const { extname, join } = require('path') const fs = require('fs') const yaml = require('js-yaml') const { v5: uuidv5 } = require('uuid') +const { mergeDeepRight } = require('ramda') const SettingsFileTypes = { yaml: 'yaml', @@ -13,18 +14,28 @@ const NAMESPACE = 'c646b451-db73-47fb-9a70-ea24ce8a225a' exports.seed = async function (knex) { const settingsFilePath = `${process.cwd()}/seeds/configs.json` - const defaultConfigs = fs.readFileSync(settingsFilePath) - // await knew.batchInsert('configs', defaultConfigs, 10) + let defaultConfigs = fs.readFileSync(settingsFilePath) + defaultConfigs = addIdsToConfigs(defaultConfigs) const rawConfigs = getConfigs() - const parsedConfigs = parseAll(rawConfigs) - if (parsedConfigs) { + const mergedSettings = mergeDeepRight(defaultConfigs, parsedConfigs) + + if (mergedSettings) { // await knex.batchInsert('configs', configsByCategory, 10) } } +const addIdsToConfigs = (configs) => { + return configs.map(config => { + return { + ...config, + id: uuidv5('key', NAMESPACE), + } + }) +} + const getConfigs = () => { const settingsFilePath = process.env.NOSTR_CONFIG_DIR ?? join(process.cwd(), '.nostr') @@ -41,7 +52,7 @@ const getConfigs = () => { if (filteredFile) { const extension = extname(filteredFile).substring(1) if (SettingsFileTypes[extension]) { - settingsFileNamePath = `${settingsFilePath}/settings.${extension}` + const settingsFileNamePath = `${settingsFilePath}/settings.${extension}` if (extension === SettingsFileTypes.json) { settingsFile = loadAndParseJsonFile(settingsFileNamePath) } else { @@ -88,7 +99,7 @@ const parseOneLevelDeepConfigs = (configs, category) => { id: uuidv5('key', NAMESPACE), key, value: configs[key], - category + category, } }) From 874d7a48566dc0b4fb3eee4429e998902d6a0ff8 Mon Sep 17 00:00:00 2001 From: antonleviathan Date: Wed, 5 Apr 2023 08:31:21 -0400 Subject: [PATCH 6/7] commit latest changes --- .../20230312_210600_create_configs_table.js | 13 ++++++ package-lock.json | 4 +- seeds/configs.json | 2 +- seeds/initial-configs.js | 39 +++++++--------- src/index.ts | 19 +++----- src/utils/settings.ts | 44 ++++++++++--------- 6 files changed, 64 insertions(+), 57 deletions(-) create mode 100644 migrations/20230312_210600_create_configs_table.js diff --git a/migrations/20230312_210600_create_configs_table.js b/migrations/20230312_210600_create_configs_table.js new file mode 100644 index 00000000..45aed174 --- /dev/null +++ b/migrations/20230312_210600_create_configs_table.js @@ -0,0 +1,13 @@ +exports.up = function (knex) { + return knex.schema.createTable('configs', (config) => { + config.unique(['key', 'category']) + config.uuid('id').primary().defaultTo(knex.raw('uuid_generate_v4()')) + config.text('key').notNullable().index() + config.jsonb('value').notNullable().index() + config.text('category').notNullable().index() + }) +} + +exports.down = function (knex) { + return knex.schema.dropTable('configs') +} diff --git a/package-lock.json b/package-lock.json index decde40f..5dc8da28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "@noble/secp256k1": "1.7.1", - "axios": "^1.2.3", + "axios": "1.2.3", "bech32": "2.0.0", "body-parser": "1.20.1", "debug": "4.3.4", @@ -20,7 +20,7 @@ "helmet": "6.0.1", "joi": "17.7.0", "js-yaml": "4.1.0", - "knex": "^2.4.1", + "knex": "2.4.1", "pg": "8.8.0", "pg-query-stream": "4.2.4", "ramda": "0.28.0", diff --git a/seeds/configs.json b/seeds/configs.json index 04d0ca6c..55f08cb0 100644 --- a/seeds/configs.json +++ b/seeds/configs.json @@ -1,7 +1,7 @@ [ { "key": "relay_url", - "value": "wss://nostream.your-domain.com", + "value": { "url": "wss://nostream.your-domain.com"}, "category": "info" }, { diff --git a/seeds/initial-configs.js b/seeds/initial-configs.js index 5aef721f..c10399c9 100644 --- a/seeds/initial-configs.js +++ b/seeds/initial-configs.js @@ -2,38 +2,33 @@ const { extname, join } = require('path') const fs = require('fs') const yaml = require('js-yaml') -const { v5: uuidv5 } = require('uuid') -const { mergeDeepRight } = require('ramda') +const { mergeDeepLeft } = require('ramda') const SettingsFileTypes = { yaml: 'yaml', json: 'json', } -const NAMESPACE = 'c646b451-db73-47fb-9a70-ea24ce8a225a' - exports.seed = async function (knex) { const settingsFilePath = `${process.cwd()}/seeds/configs.json` - let defaultConfigs = fs.readFileSync(settingsFilePath) - defaultConfigs = addIdsToConfigs(defaultConfigs) + const defaultConfigs = JSON.parse(fs.readFileSync(settingsFilePath, 'utf-8')) const rawConfigs = getConfigs() const parsedConfigs = parseAll(rawConfigs) - const mergedSettings = mergeDeepRight(defaultConfigs, parsedConfigs) - - if (mergedSettings) { - // await knex.batchInsert('configs', configsByCategory, 10) - } -} - -const addIdsToConfigs = (configs) => { - return configs.map(config => { - return { - ...config, - id: uuidv5('key', NAMESPACE), + const mergedSettings = mergeDeepLeft(defaultConfigs, parsedConfigs) + + for (const settingKey of Object.keys(mergedSettings)) { + try { + //const res = await knex('configs').insert(setting) + const res = await knex('configs').insert([mergedSettings[settingKey]]) + console.log('knex res', res) + } catch (err) { + // TODO remove this log when finished developing + console.log('Failed to insert config due to error: ', err) + // Nothing to log as if this fails the config already exists, which is fine } - }) + } } const getConfigs = () => { @@ -92,11 +87,8 @@ const parseAll = (jsonConfigs) => { } const parseOneLevelDeepConfigs = (configs, category) => { - const keys = Object.keys(configs) - console.log(keys) const flattenedConfigs = Object.keys(configs).map(key => { return { - id: uuidv5('key', NAMESPACE), key, value: configs[key], category, @@ -105,3 +97,6 @@ const parseOneLevelDeepConfigs = (configs, category) => { return flattenedConfigs } + + +// TODO: fix the key "enabled", as it repeats \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 33c71e93..75a1e116 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,38 +3,33 @@ import dotenv from 'dotenv' dotenv.config() import { appFactory } from './factories/app-factory' -import { getMasterDbClient } from './database/client' import { maintenanceWorkerFactory } from './factories/maintenance-worker-factory' import { SettingsStatic } from './utils/settings' import { staticMirroringWorkerFactory } from './factories/static-mirroring.worker-factory' import { workerFactory } from './factories/worker-factory' export const getRunner = (): any => { - const dbClient = getMasterDbClient() - const initializeSettings = new SettingsStatic(dbClient).init() - console.log('here1i') + const settingsInstance = SettingsStatic.instance - initializeSettings + settingsInstance.init() .then(() => { if (cluster.isPrimary) { - appFactory().run() + return appFactory().run() } else { switch (process.env.WORKER_TYPE) { case 'worker': - workerFactory().run() - return + return workerFactory().run() case 'maintenance': - maintenanceWorkerFactory().run() - return + return maintenanceWorkerFactory().run() case 'static-mirroring': - staticMirroringWorkerFactory().run() - return + return staticMirroringWorkerFactory().run() default: throw new Error(`Unknown worker: ${process.env.WORKER_TYPE}`) } } }) .catch(error => { + console.log('whoooops---------', error) throw new Error('Failed to load settings', error) }) } diff --git a/src/utils/settings.ts b/src/utils/settings.ts index b1344c01..67e47e43 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -1,5 +1,6 @@ import { createLogger } from '../factories/logger-factory' import { DatabaseClient } from '../@types/base' +import { getMasterDbClient } from '../database/client' import { Setting } from '../@types/setting' import { SettingRepository } from '../repositories/settings-repository' import { Settings } from '../@types/settings' @@ -9,10 +10,10 @@ const debug = createLogger('settings') export class SettingsStatic { private static _instance: SettingsStatic private static dbClient: DatabaseClient - static _settings: Settings | undefined + static _settings: any | undefined static settingsRepository: SettingRepository | undefined - constructor(dbClient: DatabaseClient) { + private constructor(dbClient: DatabaseClient) { SettingsStatic.dbClient = dbClient SettingsStatic.settingsRepository = new SettingRepository(dbClient) if (SettingsStatic._instance) { @@ -22,32 +23,34 @@ export class SettingsStatic { SettingsStatic._instance = this } - public init() { + public async init() { debug('SettingsStatic.init()') - return new Promise((resolve, reject) => { - const settingsPromise = SettingsStatic.loadSettingsFromDb(SettingsStatic.constructSettingsJsonBlob) - if (settingsPromise) { - resolve('success') - } - reject('Failed to initialize settings') - }) + await SettingsStatic.loadSettingsFromDb() + //const settingsPromise = await SettingsStatic.loadSettingsFromDb(SettingsStatic.constructSettingsJsonBlob) + //if (settingsPromise) { + // resolve('success') + //} + // reject('Failed to initialize settings') } static get instance() { - return SettingsStatic._instance ?? (SettingsStatic._instance = new SettingsStatic(this.dbClient)) + return SettingsStatic._instance ?? (SettingsStatic._instance = new SettingsStatic(getMasterDbClient())) } - private static loadSettingsFromDb(callback) { + private static async loadSettingsFromDb() { debug('SettingsStatic.loadSettingsFromDb()') - const promise = SettingsStatic.settingsRepository.getSettings() + const rawDbSettings = await SettingsStatic.settingsRepository.getSettings() + const parsedSettings = SettingsStatic.constructSettingsJsonBlob(rawDbSettings) + this._settings = parsedSettings + console.log('rawDbSettings', rawDbSettings) + console.log('parsedSettings', parsedSettings) - return promise.then(rawSettingsFromDb => { - const settingsJsonBlob = callback(rawSettingsFromDb) - this._settings = settingsJsonBlob - }) + // return promise.then(rawSettingsFromDb => { + // const settingsJsonBlob = callback(rawSettingsFromDb) + // this._settings = settingsJsonBlob + // }) } - public static createSettings(): Settings { return this._settings } @@ -59,8 +62,9 @@ export class SettingsStatic { } private static updateSingletonSettings(setting) { - const updateSettings = this._settings - updateSettings[setting.category][setting.key] = setting.value + const updatedSettings = this._settings + updatedSettings[setting.category][setting.key] = setting.value + this._settings = updatedSettings } private static constructSettingsJsonBlob(rawSettingsFromDb): any { From 76a4e79ce4bf2ecb74de59815296b6651d57927e Mon Sep 17 00:00:00 2001 From: antonleviathan Date: Sun, 9 Apr 2023 13:20:03 -0400 Subject: [PATCH 7/7] fix: seed script and settings config constructor --- seeds/configs.json | 2 +- seeds/initial-configs.js | 22 ++++++++++------------ src/utils/settings.ts | 5 ++++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/seeds/configs.json b/seeds/configs.json index 55f08cb0..04d0ca6c 100644 --- a/seeds/configs.json +++ b/seeds/configs.json @@ -1,7 +1,7 @@ [ { "key": "relay_url", - "value": { "url": "wss://nostream.your-domain.com"}, + "value": "wss://nostream.your-domain.com", "category": "info" }, { diff --git a/seeds/initial-configs.js b/seeds/initial-configs.js index c10399c9..d310fe0b 100644 --- a/seeds/initial-configs.js +++ b/seeds/initial-configs.js @@ -16,17 +16,18 @@ exports.seed = async function (knex) { const rawConfigs = getConfigs() const parsedConfigs = parseAll(rawConfigs) - const mergedSettings = mergeDeepLeft(defaultConfigs, parsedConfigs) + const rawMergedSettings = mergeDeepLeft(defaultConfigs, parsedConfigs) + const mergedSettings = Object.keys(rawMergedSettings).map(s => { + const setting = rawMergedSettings[s] + setting.value = JSON.stringify(setting.value) + return setting + }) - for (const settingKey of Object.keys(mergedSettings)) { + for (const setting of mergedSettings) { try { - //const res = await knex('configs').insert(setting) - const res = await knex('configs').insert([mergedSettings[settingKey]]) - console.log('knex res', res) + await knex('configs').insert([setting]) } catch (err) { - // TODO remove this log when finished developing - console.log('Failed to insert config due to error: ', err) - // Nothing to log as if this fails the config already exists, which is fine + console.log('Warning: failed to insert config due to error: ', err) } } } @@ -90,13 +91,10 @@ const parseOneLevelDeepConfigs = (configs, category) => { const flattenedConfigs = Object.keys(configs).map(key => { return { key, - value: configs[key], + value: JSON.stringify(configs[key]), category, } }) return flattenedConfigs } - - -// TODO: fix the key "enabled", as it repeats \ No newline at end of file diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 67e47e43..93ec57fd 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -69,7 +69,10 @@ export class SettingsStatic { private static constructSettingsJsonBlob(rawSettingsFromDb): any { const settings = {} - rawSettingsFromDb.map(setting => { + rawSettingsFromDb.forEach(setting => { + if (!settings[setting.category]) { + settings[setting.category] = {} + } settings[setting.category][setting.key] = setting.value })