From 19207f3c44ccce9d4527278c2cd6d85f8a117975 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 26 Feb 2024 13:19:37 +0100 Subject: [PATCH 01/11] feat: automatic sqs query schema generation --- .env | 10 +- src/config/app.yaml | 10 +- src/couchdb/couch-db-client.service.ts | 21 +- src/query-body.dto.ts | 14 -- .../core/entity-config-resolver.interface.ts | 10 + src/query/core/entity-config-resolver.spec.ts | 129 ++++++++++++ src/query/core/entity-config-resolver.ts | 72 +++++++ src/query/core/query-service.interface.ts | 8 + src/query/core/query-service.spec.ts | 103 ++++++++++ src/query/core/query-service.ts | 16 ++ src/query/di/query.configuration.ts | 65 ++++++ src/query/domain/EntityConfig.ts | 11 + src/query/domain/QueryRequest.ts | 3 + src/query/domain/QueryResult.ts | 3 + src/query/query.module.ts | 39 ++++ src/query/sqs/dtos.ts | 33 +++ .../sqs/sqs-schema-generator.service.spec.ts | 127 ++++++++++++ src/query/sqs/sqs-schema-generator.service.ts | 118 +++++++++++ src/query/sqs/sqs.client.spec.ts | 189 ++++++++++++++++++ src/query/sqs/sqs.client.ts | 48 +++++ .../di/report-changes-configuration.ts | 2 +- ...c.ts => report-calculator.service.spec.ts} | 12 +- ...ervice.ts => report-calculator.service.ts} | 17 +- src/report/core/report-storage.interface.ts | 2 +- src/report/di/report-configuration.ts | 52 ++--- src/report/report.module.ts | 20 +- .../sqs/couch-sqs-client.service.spec.ts | 18 -- src/report/sqs/couch-sqs.client.ts | 43 ---- .../storage/reporting-storage.service.spec.ts | 6 +- .../storage/reporting-storage.service.ts | 4 +- ...port-calculation-processor.service.spec.ts | 4 +- .../report-calculation-processor.service.ts | 11 +- src/sql-report.ts | 8 - 33 files changed, 1041 insertions(+), 187 deletions(-) delete mode 100644 src/query-body.dto.ts create mode 100644 src/query/core/entity-config-resolver.interface.ts create mode 100644 src/query/core/entity-config-resolver.spec.ts create mode 100644 src/query/core/entity-config-resolver.ts create mode 100644 src/query/core/query-service.interface.ts create mode 100644 src/query/core/query-service.spec.ts create mode 100644 src/query/core/query-service.ts create mode 100644 src/query/di/query.configuration.ts create mode 100644 src/query/domain/EntityConfig.ts create mode 100644 src/query/domain/QueryRequest.ts create mode 100644 src/query/domain/QueryResult.ts create mode 100644 src/query/query.module.ts create mode 100644 src/query/sqs/dtos.ts create mode 100644 src/query/sqs/sqs-schema-generator.service.spec.ts create mode 100644 src/query/sqs/sqs-schema-generator.service.ts create mode 100644 src/query/sqs/sqs.client.spec.ts create mode 100644 src/query/sqs/sqs.client.ts rename src/report/core/{sqs-report-calculator.service.spec.ts => report-calculator.service.spec.ts} (64%) rename src/report/core/{sqs-report-calculator.service.ts => report-calculator.service.ts} (75%) delete mode 100644 src/report/sqs/couch-sqs-client.service.spec.ts delete mode 100644 src/report/sqs/couch-sqs.client.ts delete mode 100644 src/sql-report.ts diff --git a/.env b/.env index 7553edf..3f9d826 100644 --- a/.env +++ b/.env @@ -2,14 +2,16 @@ ; CouchDb Basic Auth credentials for different clients COUCH_DB_CLIENT_NOTIFICATION_BASIC_AUTH_USER=admin COUCH_DB_CLIENT_NOTIFICATION_BASIC_AUTH_PASSWORD=docker -COUCH_DB_CLIENT_REPORT_BASIC_AUTH_USER=admin -COUCH_DB_CLIENT_REPORT_BASIC_AUTH_PASSWORD=docker + +COUCH_DB_CLIENT_APP_BASIC_AUTH_USER=admin +COUCH_DB_CLIENT_APP_BASIC_AUTH_PASSWORD=docker + COUCH_DB_CLIENT_REPORT_CALCULATION_BASIC_AUTH_USER=admin COUCH_DB_CLIENT_REPORT_CALCULATION_BASIC_AUTH_PASSWORD=docker ; SQS Basic Auth credentials -SQS_CLIENT_BASIC_AUTH_USER=admin -SQS_CLIENT_BASIC_AUTH_PASSWORD=docker +QUERY_SQS_CLIENT_BASIC_AUTH_USER=admin +QUERY_SQS_CLIENT_BASIC_AUTH_PASSWORD=docker ; Encryption Key for server side secret encryption ; e.g. webhook credentials stored encrypted in database diff --git a/src/config/app.yaml b/src/config/app.yaml index 83f5180..a5f928e 100644 --- a/src/config/app.yaml +++ b/src/config/app.yaml @@ -5,7 +5,7 @@ COUCH_DB_CLIENT: NOTIFICATION: BASE_URL: http://localhost:5984 TARGET_DATABASE: notification-webhook - REPORT: + APP: BASE_URL: http://localhost:5984 TARGET_DATABASE: app REPORT_CALCULATION: @@ -15,8 +15,10 @@ COUCH_DB_CLIENT: # CouchDb SQS client config # BASE_URL: URL of the SQS # SCHEMA_DESIGN_CONFIG: database ID of the document which holds the SQS schema (default `/app/_design/sqlite:config`) -SQS_CLIENT: - BASE_URL: http://localhost:4984 - SCHEMA_DESIGN_CONFIG: /app/_design/sqlite:config +QUERY: + SCHEMA_DESIGN_CONFIG: /_design/sqlite:config + SQS_CLIENT: + BASE_URL: http://localhost:4984 + TARGET_DATABASE: app REPORT_CHANGES_POLL_INTERVAL: 10000 diff --git a/src/couchdb/couch-db-client.service.ts b/src/couchdb/couch-db-client.service.ts index a8fd87f..fbd226f 100644 --- a/src/couchdb/couch-db-client.service.ts +++ b/src/couchdb/couch-db-client.service.ts @@ -1,10 +1,4 @@ -import { - ForbiddenException, - InternalServerErrorException, - Logger, - NotFoundException, - UnauthorizedException, -} from '@nestjs/common'; +import { Logger } from '@nestjs/common'; import { catchError, map, Observable, of, switchMap } from 'rxjs'; import { HttpService } from '@nestjs/axios'; import { AxiosHeaders } from 'axios'; @@ -131,17 +125,6 @@ export class CouchDbClient { } private handleError(err: any) { - console.error(err); - - if (err.response?.status === 401) { - throw new UnauthorizedException(); - } - if (err.response?.status === 403) { - throw new ForbiddenException(); - } - if (err.response?.status === 404) { - throw new NotFoundException(); - } - throw new InternalServerErrorException(); + this.logger.error(err); } } diff --git a/src/query-body.dto.ts b/src/query-body.dto.ts deleted file mode 100644 index 0136960..0000000 --- a/src/query-body.dto.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * The dates can be used in the SQL SELECT statements with a "?" - * "from" will replace the first "?" - * "to" will replace the second "?" - */ -export class QueryBody { - from: string; - to: string; - - constructor(from: string, to: string) { - this.from = from; - this.to = to; - } -} diff --git a/src/query/core/entity-config-resolver.interface.ts b/src/query/core/entity-config-resolver.interface.ts new file mode 100644 index 0000000..617c006 --- /dev/null +++ b/src/query/core/entity-config-resolver.interface.ts @@ -0,0 +1,10 @@ +import { Observable } from 'rxjs'; +import { EntityConfig } from '../domain/EntityConfig'; + +/** + * EntityConfigResolver + * Notice: Could be moved to a separate module for handling configuration + */ +export interface IEntityConfigResolver { + getEntityConfig(): Observable; +} diff --git a/src/query/core/entity-config-resolver.spec.ts b/src/query/core/entity-config-resolver.spec.ts new file mode 100644 index 0000000..16c706e --- /dev/null +++ b/src/query/core/entity-config-resolver.spec.ts @@ -0,0 +1,129 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { + AppConfigFile, + EntityConfigResolver, + EntityConfigResolverConfig, +} from './entity-config-resolver'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { of } from 'rxjs'; + +describe('EntityConfigResolver', () => { + let service: EntityConfigResolver; + + let mockCouchDbClient: { + getDatabaseDocument: jest.Mock; + }; + + beforeEach(async () => { + mockCouchDbClient = { + getDatabaseDocument: jest.fn(), + }; + + const module: TestingModule = await Test.createTestingModule({ + imports: [], + providers: [ + { + provide: EntityConfigResolver, + useFactory: (couchDbClient) => { + const config = new EntityConfigResolverConfig(); + config.FILENAME_CONFIG_ENTITY = 'foo.config'; + return new EntityConfigResolver(couchDbClient, config); + }, + inject: [CouchDbClient], + }, + { + provide: CouchDbClient, + useValue: mockCouchDbClient, + }, + ], + }).compile(); + + service = module.get(EntityConfigResolver); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('getEntityConfig() should return EntityConfig with all entities from foo.config', (done) => { + jest.spyOn(mockCouchDbClient, 'getDatabaseDocument').mockReturnValue( + of({ + _id: 'id-1', + _rev: 'rev-1', + data: { + 'view:v1': { + label: 'v-1', + attributes: {}, + }, + 'entity:conf-1': { + label: 'conf-1', + attributes: { + 'att-1': { + dataType: 'TEXT', + }, + 'att-2': { + dataType: 'INTEGER', + }, + }, + }, + 'entity:conf-2': { + attributes: { + 'att-21': { + dataType: 'TEXT', + }, + 'att-22': { + dataType: 'INTEGER', + }, + }, + }, + }, + } as AppConfigFile), + ); + + service.getEntityConfig().subscribe({ + next: (value) => { + expect(mockCouchDbClient.getDatabaseDocument).toHaveBeenCalledWith({ + documentId: 'foo.config', + config: {}, + }); + + expect(value).toEqual({ + version: 'rev-1', + entities: [ + { + label: 'conf-1', + attributes: [ + { + name: 'att-1', + type: 'TEXT', + }, + { + name: 'att-2', + type: 'INTEGER', + }, + ], + }, + { + label: 'conf-2', + attributes: [ + { + name: 'att-21', + type: 'TEXT', + }, + { + name: 'att-22', + type: 'INTEGER', + }, + ], + }, + ], + }); + + done(); + }, + error: (err) => { + done(err); + }, + }); + }); +}); diff --git a/src/query/core/entity-config-resolver.ts b/src/query/core/entity-config-resolver.ts new file mode 100644 index 0000000..87702cf --- /dev/null +++ b/src/query/core/entity-config-resolver.ts @@ -0,0 +1,72 @@ +import { IEntityConfigResolver } from './entity-config-resolver.interface'; +import { map, Observable } from 'rxjs'; +import { Entity, EntityAttribute, EntityConfig } from '../domain/EntityConfig'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; + +export interface AppConfigFile { + _id: string; + _rev: string; + data: { + [key: string]: { + label?: string; + attributes: { + [key: string]: { + dataType: string; + }; + }; + }; + }; +} + +export class EntityConfigResolverConfig { + FILENAME_CONFIG_ENTITY = ''; +} + +export class EntityConfigResolver implements IEntityConfigResolver { + constructor( + private couchDbClient: CouchDbClient, + private config: EntityConfigResolverConfig, + ) {} + + getEntityConfig(): Observable { + return this.couchDbClient + .getDatabaseDocument({ + documentId: this.config.FILENAME_CONFIG_ENTITY, + config: {}, + }) + .pipe( + map((config) => { + const keys = Object.keys(config.data).filter((key) => + key.startsWith('entity:'), + ); + const entities: Entity[] = []; + for (let i = 0; i < keys.length; i++) { + entities.push(this.parseEntityConfig(keys[i], config)); + } + return new EntityConfig(config._rev, entities); + }), + ); + } + + private parseEntityConfig(entityKey: string, config: AppConfigFile): Entity { + const data = config.data[entityKey]; + + let label: string; + + if (data.label) { + label = data.label; + } else { + label = entityKey.split(':')[1]; + } + + const attributes: EntityAttribute[] = Object.keys(data.attributes).map( + (attributeKey) => + new EntityAttribute( + attributeKey, + data.attributes[attributeKey].dataType, + ), + ); + + return new Entity(label, attributes); + } +} diff --git a/src/query/core/query-service.interface.ts b/src/query/core/query-service.interface.ts new file mode 100644 index 0000000..2c3e4bc --- /dev/null +++ b/src/query/core/query-service.interface.ts @@ -0,0 +1,8 @@ +import { QueryRequest } from '../domain/QueryRequest'; +import { Observable } from 'rxjs'; +import { QueryResult } from '../domain/QueryResult'; + +export interface IQueryService { + executeQuery(query: QueryRequest): Observable; + executeQueries(queries: QueryRequest[]): Observable; +} diff --git a/src/query/core/query-service.spec.ts b/src/query/core/query-service.spec.ts new file mode 100644 index 0000000..254670f --- /dev/null +++ b/src/query/core/query-service.spec.ts @@ -0,0 +1,103 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { QueryService } from './query-service'; +import { SqsClient } from '../sqs/sqs.client'; +import { of } from 'rxjs'; +import { QueryRequest } from '../domain/QueryRequest'; +import { QueryResult } from '../domain/QueryResult'; + +describe('QueryService', () => { + let service: QueryService; + + let mockSqsClient: { + executeQuery: jest.Mock; + executeQueries: jest.Mock; + }; + + beforeEach(async () => { + mockSqsClient = { + executeQuery: jest.fn(), + executeQueries: jest.fn(), + }; + + const module: TestingModule = await Test.createTestingModule({ + imports: [], + providers: [ + { + provide: QueryService, + useFactory: (sqsClient: SqsClient) => { + return new QueryService(sqsClient); + }, + inject: [SqsClient], + }, + { + provide: SqsClient, + useValue: mockSqsClient, + }, + ], + }).compile(); + + service = module.get(QueryService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('executeQuery() should call sqsClient with query', (done) => { + jest + .spyOn(mockSqsClient, 'executeQuery') + .mockReturnValue(of(new QueryResult('foo bar do result'))); + + service.executeQuery(new QueryRequest('foo bar do')).subscribe({ + next: (value) => { + expect(mockSqsClient.executeQuery).toHaveBeenCalledWith({ + query: 'foo bar do', + }); + + expect(value).toEqual({ + result: 'foo bar do result', + }); + + done(); + }, + error: (err) => { + done(err); + }, + }); + }); + + it('executeQueries() should call sqsClient with queries', (done) => { + jest + .spyOn(mockSqsClient, 'executeQueries') + .mockReturnValue( + of([ + new QueryResult('foo bar do result'), + new QueryResult('do bar foo result'), + ]), + ); + + service + .executeQueries([ + new QueryRequest('foo bar do'), + new QueryRequest('do bar foo'), + ]) + .subscribe({ + next: (value) => { + expect(mockSqsClient.executeQueries).toHaveBeenCalledWith([ + { query: 'foo bar do' }, + { query: 'do bar foo' }, + ]); + + expect(value).toEqual([ + { result: 'foo bar do result' }, + { result: 'do bar foo result' }, + ]); + + done(); + }, + error: (err) => { + done(err); + }, + }); + }); +}); diff --git a/src/query/core/query-service.ts b/src/query/core/query-service.ts new file mode 100644 index 0000000..f946ad3 --- /dev/null +++ b/src/query/core/query-service.ts @@ -0,0 +1,16 @@ +import { Observable } from 'rxjs'; +import { QueryRequest } from '../domain/QueryRequest'; +import { QueryResult } from '../domain/QueryResult'; +import { IQueryService } from './query-service.interface'; +import { SqsClient } from '../sqs/sqs.client'; + +export class QueryService implements IQueryService { + constructor(private sqsClient: SqsClient) {} + + executeQuery(query: QueryRequest): Observable { + return this.sqsClient.executeQuery(query); + } + executeQueries(queries: QueryRequest[]): Observable { + return this.sqsClient.executeQueries(queries); + } +} diff --git a/src/query/di/query.configuration.ts b/src/query/di/query.configuration.ts new file mode 100644 index 0000000..7bf3b2d --- /dev/null +++ b/src/query/di/query.configuration.ts @@ -0,0 +1,65 @@ +import { ConfigService } from '@nestjs/config'; +import { CouchSqsClientConfig, SqsClient } from '../sqs/sqs.client'; +import axios from 'axios'; +import { HttpService } from '@nestjs/axios'; +import { SqsSchemaService } from '../sqs/sqs-schema-generator.service'; +import { DefaultCouchDbClientFactory } from '../../couchdb/default-factory'; +import { EntityConfigResolver } from '../core/entity-config-resolver'; +import { QueryService } from '../core/query-service'; +import { IEntityConfigResolver } from '../core/entity-config-resolver.interface'; + +export const SqsClientFactory = ( + configService: ConfigService, + sqsSchemaService: SqsSchemaService, +): SqsClient => { + const CONFIG_PREFIX = 'QUERY_SQS_CLIENT_'; + + const couchSqsClientConfig: CouchSqsClientConfig = { + BASE_URL: configService.getOrThrow(CONFIG_PREFIX + 'BASE_URL'), + BASIC_AUTH_USER: configService.getOrThrow( + CONFIG_PREFIX + 'BASIC_AUTH_USER', + ), + BASIC_AUTH_PASSWORD: configService.getOrThrow( + CONFIG_PREFIX + 'BASIC_AUTH_PASSWORD', + ), + }; + + const axiosInstance = axios.create(); + + axiosInstance.defaults.baseURL = `${ + couchSqsClientConfig.BASE_URL + }/${configService.getOrThrow(CONFIG_PREFIX + 'TARGET_DATABASE')}`; + axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( + `${couchSqsClientConfig.BASIC_AUTH_USER}:${couchSqsClientConfig.BASIC_AUTH_PASSWORD}`, + ).toString('base64')}`; + + return new SqsClient(new HttpService(axiosInstance), sqsSchemaService); +}; + +export const SqsSchemaServiceFactory = ( + configService: ConfigService, + entityConfigResolver: IEntityConfigResolver, +): SqsSchemaService => { + const couchDbClient = DefaultCouchDbClientFactory( + 'COUCH_DB_CLIENT_APP_', + configService, + ); + return new SqsSchemaService(couchDbClient, entityConfigResolver, { + SCHEMA_PATH: configService.getOrThrow('QUERY_SCHEMA_DESIGN_CONFIG'), + }); +}; + +export const QueryServiceFactory = (sqsClient: SqsClient): QueryService => + new QueryService(sqsClient); + +export const EntityConfigResolverFactory = ( + configService: ConfigService, +): EntityConfigResolver => { + const couchDbClient = DefaultCouchDbClientFactory( + 'COUCH_DB_CLIENT_APP_', + configService, + ); + return new EntityConfigResolver(couchDbClient, { + FILENAME_CONFIG_ENTITY: 'Config:CONFIG_ENTITY', + }); +}; diff --git a/src/query/domain/EntityConfig.ts b/src/query/domain/EntityConfig.ts new file mode 100644 index 0000000..f8ab773 --- /dev/null +++ b/src/query/domain/EntityConfig.ts @@ -0,0 +1,11 @@ +export class EntityAttribute { + constructor(public name: string, public type: string) {} +} + +export class Entity { + constructor(public label: string, public attributes: EntityAttribute[]) {} +} + +export class EntityConfig { + constructor(public version: string, public entities: Entity[]) {} +} diff --git a/src/query/domain/QueryRequest.ts b/src/query/domain/QueryRequest.ts new file mode 100644 index 0000000..f810367 --- /dev/null +++ b/src/query/domain/QueryRequest.ts @@ -0,0 +1,3 @@ +export class QueryRequest { + constructor(public query: string) {} +} diff --git a/src/query/domain/QueryResult.ts b/src/query/domain/QueryResult.ts new file mode 100644 index 0000000..70f9f3b --- /dev/null +++ b/src/query/domain/QueryResult.ts @@ -0,0 +1,3 @@ +export class QueryResult { + constructor(public result: any) {} +} diff --git a/src/query/query.module.ts b/src/query/query.module.ts new file mode 100644 index 0000000..fb1bb7b --- /dev/null +++ b/src/query/query.module.ts @@ -0,0 +1,39 @@ +import { Module } from '@nestjs/common'; +import { SqsSchemaService } from './sqs/sqs-schema-generator.service'; +import { SqsClient } from './sqs/sqs.client'; +import { ConfigService } from '@nestjs/config'; +import { + EntityConfigResolverFactory, + QueryServiceFactory, + SqsClientFactory, + SqsSchemaServiceFactory, +} from './di/query.configuration'; +import { QueryService } from './core/query-service'; +import { EntityConfigResolver } from './core/entity-config-resolver'; + +@Module({ + providers: [ + { + provide: SqsSchemaService, + useFactory: SqsSchemaServiceFactory, + inject: [ConfigService, EntityConfigResolver], + }, + { + provide: SqsClient, + useFactory: SqsClientFactory, + inject: [ConfigService, SqsSchemaService], + }, + { + provide: QueryService, + useFactory: QueryServiceFactory, + inject: [SqsClient], + }, + { + provide: EntityConfigResolver, + useFactory: EntityConfigResolverFactory, + inject: [ConfigService], + }, + ], + exports: [QueryService], +}) +export class QueryModule {} diff --git a/src/query/sqs/dtos.ts b/src/query/sqs/dtos.ts new file mode 100644 index 0000000..ad52b82 --- /dev/null +++ b/src/query/sqs/dtos.ts @@ -0,0 +1,33 @@ +export class SqsSchema { + constructor( + public sql: { + // SQL table definitions + tables: SqlTables; + // Optional SQL indices + indexes?: string[]; + // Further options + options?: SqlOptions; + }, + public language: 'sqlite' = 'sqlite', + ) {} +} + +type SqlTables = { + // Name of the entity + [table: string]: { + fields: { + // Name of the entity attribute and the type of it + [column: string]: SqlType | { field: string; type: SqlType }; + }; + }; +}; + +type SqlType = 'TEXT' | 'INTEGER' | 'REAL' | 'JSON'; + +type SqlOptions = { + table_name: { + operation: 'prefix'; + field: string; + separator: string; + }; +}; diff --git a/src/query/sqs/sqs-schema-generator.service.spec.ts b/src/query/sqs/sqs-schema-generator.service.spec.ts new file mode 100644 index 0000000..dcbc349 --- /dev/null +++ b/src/query/sqs/sqs-schema-generator.service.spec.ts @@ -0,0 +1,127 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { SqsSchemaService } from './sqs-schema-generator.service'; + +describe('SchemaGeneratorService', () => { + let service: SqsSchemaService; + + const entityConfig = { + 'entity:Child': { + label: 'Child', + labelPlural: 'Children', + attributes: { + address: { + dataType: 'location', + label: 'Address', + }, + health_bloodGroup: { + dataType: 'string', + label: 'Blood Group', + }, + religion: { + dataType: 'string', + label: 'Religion', + }, + motherTongue: { + dataType: 'string', + label: 'Mother Tongue', + description: 'The primary language spoken at home', + }, + health_lastDentalCheckup: { + dataType: 'date', + label: 'Last Dental Check-Up', + }, + birth_certificate: { + dataType: 'file', + label: 'Birth certificate', + }, + }, + }, + 'entity:School': { + attributes: { + name: { + dataType: 'string', + label: 'Name', + }, + privateSchool: { + dataType: 'boolean', + label: 'Private School', + }, + language: { + dataType: 'string', + label: 'Language', + }, + address: { + dataType: 'location', + label: 'Address', + }, + phone: { + dataType: 'string', + label: 'Phone Number', + }, + timing: { + dataType: 'string', + label: 'School Timing', + }, + remarks: { + dataType: 'string', + label: 'Remarks', + }, + }, + }, + 'entity:HistoricalEntityData': { + attributes: { + isMotivatedDuringClass: { + dataType: 'configurable-enum', + additional: 'rating-answer', + label: 'Motivated', + description: 'The child is motivated during the class.', + }, + isParticipatingInClass: { + dataType: 'configurable-enum', + additional: 'rating-answer', + label: 'Participating', + description: 'The child is actively participating in the class.', + }, + isInteractingWithOthers: { + dataType: 'configurable-enum', + additional: 'rating-answer', + label: 'Interacting', + description: + 'The child interacts with other students during the class.', + }, + doesHomework: { + dataType: 'configurable-enum', + additional: 'rating-answer', + label: 'Homework', + description: 'The child does its homework.', + }, + asksQuestions: { + dataType: 'configurable-enum', + additional: 'rating-answer', + label: 'Asking Questions', + description: 'The child is asking questions during the class.', + }, + }, + }, + 'entity:User': { + attributes: { + phone: { + dataType: 'string', + label: 'Contact', + }, + }, + }, + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [SqsSchemaService], + }).compile(); + + service = module.get(SqsSchemaService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/query/sqs/sqs-schema-generator.service.ts b/src/query/sqs/sqs-schema-generator.service.ts new file mode 100644 index 0000000..ecf3bc2 --- /dev/null +++ b/src/query/sqs/sqs-schema-generator.service.ts @@ -0,0 +1,118 @@ +import { EMPTY, map, Observable, switchMap } from 'rxjs'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { IEntityConfigResolver } from '../core/entity-config-resolver.interface'; +import { SqsSchema } from './dtos'; +import { DocSuccess } from '../../couchdb/dtos'; +import { EntityAttribute, EntityConfig } from '../domain/EntityConfig'; + +export class SqsSchemaGeneratorConfig { + SCHEMA_PATH = ''; +} + +export class SqsSchemaService { + private _entityConfigVersion = ''; + + constructor( + private couchDbClient: CouchDbClient, + private entityConfigResolver: IEntityConfigResolver, + private config: SqsSchemaGeneratorConfig, + ) {} + + getSchemaPath(): string { + return this.config.SCHEMA_PATH; + } + + /** + * Loads EntityConfig and updates SqsSchema if necessary + */ + updateSchema(): Observable { + return this.entityConfigResolver.getEntityConfig().pipe( + switchMap((entityConfig) => { + if (entityConfig.version === this._entityConfigVersion) { + return EMPTY; + } else { + return this.couchDbClient + .putDatabaseDocument({ + documentId: this.config.SCHEMA_PATH, + body: this.mapToSqsSchema(entityConfig), + config: {}, + }) + .pipe( + map((result) => { + this._entityConfigVersion = result.rev; + }), + ); + } + }), + ); + } + + private mapToSqsSchema(entityConfig: EntityConfig): SqsSchema { + const sqsSchema = new SqsSchema({ + tables: {}, + indexes: [], + options: { + table_name: { + operation: 'prefix', + field: '_id', + separator: ':', + }, + }, + }); + + entityConfig.entities.forEach((entityConfig) => { + const fields: { + [column: string]: 'TEXT' | 'INTEGER'; + } = {}; + + entityConfig.attributes.forEach((ea) => { + if (!this.ignoreDataType(ea.type)) { + fields[ea.name] = this.mapConfigDataTypeToSqsDataType(ea.type); + } + }); + + this.getDefaultEntityAttributes().forEach((ea) => { + if (fields[ea.name] === undefined && !this.ignoreDataType(ea.type)) { + fields[ea.name] = this.mapConfigDataTypeToSqsDataType(ea.type); + } + }); + + sqsSchema.sql.tables[entityConfig.label] = { + fields: fields, + }; + }); + + return sqsSchema; + } + + private getDefaultEntityAttributes(): EntityAttribute[] { + return [ + { name: '_id', type: 'TEXT' }, + { name: '_rev', type: 'TEXT' }, + { name: 'created', type: 'TEXT' }, + { name: 'updated', type: 'TEXT' }, + { name: 'inactive', type: 'INTEGER' }, + { name: 'anonymized', type: 'INTEGER' }, + ]; + } + + private mapConfigDataTypeToSqsDataType(dataType: string): 'TEXT' | 'INTEGER' { + switch (dataType.toLowerCase()) { + case 'boolean': + case 'number': + case 'integer': + return 'INTEGER'; + default: + return 'TEXT'; + } + } + + private ignoreDataType(dataType: string): boolean { + switch (dataType) { + case 'file': + return true; + default: + return false; + } + } +} diff --git a/src/query/sqs/sqs.client.spec.ts b/src/query/sqs/sqs.client.spec.ts new file mode 100644 index 0000000..f517d2c --- /dev/null +++ b/src/query/sqs/sqs.client.spec.ts @@ -0,0 +1,189 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { SqsClient } from './sqs.client'; +import { HttpService } from '@nestjs/axios'; +import { SqsSchemaService } from './sqs-schema-generator.service'; +import { of, throwError } from 'rxjs'; +import { Logger } from '@nestjs/common'; + +describe('SqsClient', () => { + let service: SqsClient; + + let mockSqsSchemaService: { + getSchemaPath: jest.Mock; + updateSchema: jest.Mock; + }; + + let mockHttp: { post: jest.Mock; axiosRef: any }; + + let mockLogger: { error: jest.Mock; debug: jest.Mock }; + + beforeEach(async () => { + mockSqsSchemaService = { + getSchemaPath: jest.fn(), + updateSchema: jest.fn(), + }; + + mockHttp = { + post: jest.fn(), + axiosRef: { + defaults: { + url: 'doo', + }, + }, + }; + + mockLogger = { + error: jest.fn(), + debug: jest.fn(), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + { + provide: SqsClient, + useFactory: (http, sqsSchemaService) => + new SqsClient(http, sqsSchemaService), + inject: [HttpService, SqsSchemaService], + }, + { provide: HttpService, useValue: mockHttp }, + { provide: SqsSchemaService, useValue: mockSqsSchemaService }, + { provide: Logger, useValue: mockLogger }, + ], + }).compile(); + + service = module.get(SqsClient); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('executeQuery() should execute query and return QueryResult', (done) => { + jest + .spyOn(mockSqsSchemaService, 'getSchemaPath') + .mockReturnValue('/app/config_path'); + + jest + .spyOn(mockSqsSchemaService, 'updateSchema') + .mockReturnValue(of(undefined)); + + jest.spyOn(mockHttp, 'post').mockReturnValue( + of({ + data: { + foo: 'bar', + }, + }), + ); + + service + .executeQuery({ + query: 'SELECT foo FROM bar', + }) + .subscribe({ + next: (value) => { + expect(value).toEqual({ + result: { + foo: 'bar', + }, + }); + + expect(mockLogger.error).not.toBeCalled(); + expect(mockLogger.debug).not.toBeCalled(); + + done(); + }, + error: (err) => { + done(err); + }, + }); + }); + + it('executeQuery() should handle error from httpService', (done) => { + jest + .spyOn(mockSqsSchemaService, 'getSchemaPath') + .mockReturnValue('/app/config_path'); + + jest + .spyOn(mockSqsSchemaService, 'updateSchema') + .mockReturnValue(of(undefined)); + + jest + .spyOn(mockHttp, 'post') + .mockReturnValue(throwError(() => new Error('foo error'))); + + service + .executeQuery({ + query: 'SELECT foo FROM bar', + }) + .subscribe({ + next: (value) => { + done('should throw error '); + }, + error: (err) => { + expect(err.message).toBe('foo error'); + expect(mockLogger.error).not.toBeCalled(); + done(); + }, + }); + }); + + it('executeQueries() should execute all queries and return QueryResult[]', (done) => { + jest + .spyOn(mockSqsSchemaService, 'getSchemaPath') + .mockReturnValue('/app/config_path'); + + jest + .spyOn(mockSqsSchemaService, 'updateSchema') + .mockReturnValue(of(undefined)); + + jest.spyOn(mockHttp, 'post').mockReturnValueOnce( + of({ + data: { + foo: 'bar', + }, + }), + ); + + jest.spyOn(mockHttp, 'post').mockReturnValueOnce( + of({ + data: { + bar: 'doo', + }, + }), + ); + + service + .executeQueries([ + { + query: 'SELECT foo FROM bar', + }, + { + query: 'SELECT foo FROM bar', + }, + ]) + .subscribe({ + next: (value) => { + expect(value).toEqual([ + { + result: { + foo: 'bar', + }, + }, + { + result: { + bar: 'doo', + }, + }, + ]); + + expect(mockLogger.error).not.toBeCalled(); + expect(mockLogger.debug).not.toBeCalled(); + + done(); + }, + error: (err) => { + done(err); + }, + }); + }); +}); diff --git a/src/query/sqs/sqs.client.ts b/src/query/sqs/sqs.client.ts new file mode 100644 index 0000000..d07e361 --- /dev/null +++ b/src/query/sqs/sqs.client.ts @@ -0,0 +1,48 @@ +import { Logger } from '@nestjs/common'; +import { HttpService } from '@nestjs/axios'; +import { catchError, forkJoin, map, Observable, switchMap } from 'rxjs'; +import { QueryRequest } from '../domain/QueryRequest'; +import { QueryResult } from '../domain/QueryResult'; +import { SqsSchemaService } from './sqs-schema-generator.service'; + +export class CouchSqsClientConfig { + BASE_URL = ''; + BASIC_AUTH_USER = ''; + BASIC_AUTH_PASSWORD = ''; +} + +export class SqsClient { + private readonly logger: Logger = new Logger(SqsClient.name); + + constructor( + private httpService: HttpService, + private schemaService: SqsSchemaService, + ) {} + + executeQuery(query: QueryRequest): Observable { + const schemaPath = this.schemaService.getSchemaPath(); + return this.schemaService.updateSchema().pipe( + switchMap(() => + this.httpService.post(schemaPath, query).pipe( + map((response) => new QueryResult(response.data)), + catchError((err) => { + this.logger.error(err); + this.logger.debug( + '[CouchSqsClient] Could not execute Query: ', + this.httpService.axiosRef.defaults.url, + schemaPath, + query, + ); + throw err; + }), + ), + ), + ); + } + + executeQueries(queries: QueryRequest[]): Observable { + return forkJoin( + queries.map((queryRequest) => this.executeQuery(queryRequest)), + ); + } +} diff --git a/src/report-changes/di/report-changes-configuration.ts b/src/report-changes/di/report-changes-configuration.ts index 708e73e..b8fa796 100644 --- a/src/report-changes/di/report-changes-configuration.ts +++ b/src/report-changes/di/report-changes-configuration.ts @@ -11,7 +11,7 @@ export const CouchdbChangesServiceFactory = ( configService: ConfigService, ): CouchDbChangesService => { return new CouchDbChangesService( - DefaultCouchDbClientFactory('COUCH_DB_CLIENT_REPORT_', configService), + DefaultCouchDbClientFactory('COUCH_DB_CLIENT_APP_', configService), { POLL_INTERVAL: configService.getOrThrow('REPORT_CHANGES_POLL_INTERVAL'), }, diff --git a/src/report/core/sqs-report-calculator.service.spec.ts b/src/report/core/report-calculator.service.spec.ts similarity index 64% rename from src/report/core/sqs-report-calculator.service.spec.ts rename to src/report/core/report-calculator.service.spec.ts index ddde523..fa1964c 100644 --- a/src/report/core/sqs-report-calculator.service.spec.ts +++ b/src/report/core/report-calculator.service.spec.ts @@ -1,10 +1,10 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { SqsReportCalculator } from './sqs-report-calculator.service'; +import { ReportCalculator } from './report-calculator.service'; import { ReportingStorage } from '../storage/reporting-storage.service'; -import { CouchSqsClient } from '../sqs/couch-sqs.client'; +import { SqsClient } from '../../query/sqs/sqs.client'; describe('SqsReportCalculatorService', () => { - let service: SqsReportCalculator; + let service: ReportCalculator; let mockCouchSqsClient: { executeQuery: jest.Mock }; let mockReportStorage: { fetchAllReports: jest.Mock }; @@ -12,13 +12,13 @@ describe('SqsReportCalculatorService', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ providers: [ - SqsReportCalculator, - { provide: CouchSqsClient, useValue: mockCouchSqsClient }, + ReportCalculator, + { provide: SqsClient, useValue: mockCouchSqsClient }, { provide: ReportingStorage, useValue: mockReportStorage }, ], }).compile(); - service = module.get(SqsReportCalculator); + service = module.get(ReportCalculator); }); it('should be defined', () => { diff --git a/src/report/core/sqs-report-calculator.service.ts b/src/report/core/report-calculator.service.ts similarity index 75% rename from src/report/core/sqs-report-calculator.service.ts rename to src/report/core/report-calculator.service.ts index 8c80dbd..18e32d4 100644 --- a/src/report/core/sqs-report-calculator.service.ts +++ b/src/report/core/report-calculator.service.ts @@ -7,15 +7,15 @@ import { IReportCalculator } from './report-calculator.interface'; import { ReportData } from '../../domain/report-data'; import { map, mergeAll, Observable, switchMap } from 'rxjs'; import { ReportCalculation } from '../../domain/report-calculation'; -import { ReportingStorage } from '../storage/reporting-storage.service'; -import { CouchSqsClient } from '../sqs/couch-sqs.client'; import { v4 as uuidv4 } from 'uuid'; import { Reference } from '../../domain/reference'; +import { IReportingStorage } from './report-storage.interface'; +import { IQueryService } from '../../query/core/query-service.interface'; -export class SqsReportCalculator implements IReportCalculator { +export class ReportCalculator implements IReportCalculator { constructor( - private sqsClient: CouchSqsClient, - private reportStorage: ReportingStorage, + private queryService: IQueryService, + private reportStorage: IReportingStorage, ) {} calculate(reportCalculation: ReportCalculation): Observable { @@ -34,18 +34,17 @@ export class SqsReportCalculator implements IReportCalculator { } return report.queries.flatMap((query) => { - return this.sqsClient + return this.queryService .executeQuery({ query: query, - args: [], // TODO pass args here }) .pipe( - map((rawResponse) => { + map((queryResult) => { return new ReportData( `ReportData:${uuidv4()}`, reportCalculation.report, new Reference(reportCalculation.id), - ).setData(rawResponse); + ).setData(queryResult.result); }), ); }); diff --git a/src/report/core/report-storage.interface.ts b/src/report/core/report-storage.interface.ts index 1d3f950..7dadb1a 100644 --- a/src/report/core/report-storage.interface.ts +++ b/src/report/core/report-storage.interface.ts @@ -4,7 +4,7 @@ import { Observable, Subject } from 'rxjs'; import { ReportCalculation } from '../../domain/report-calculation'; import { ReportData } from '../../domain/report-data'; -export interface IReportStorage { +export interface IReportingStorage { fetchAllReports(authToken: string, mode: string): Observable; fetchReport( diff --git a/src/report/di/report-configuration.ts b/src/report/di/report-configuration.ts index 2d146bd..4af2813 100644 --- a/src/report/di/report-configuration.ts +++ b/src/report/di/report-configuration.ts @@ -1,49 +1,22 @@ import { DefaultCouchDbClientFactory } from '../../couchdb/default-factory'; import { ConfigService } from '@nestjs/config'; -import { CouchSqsClient, CouchSqsClientConfig } from '../sqs/couch-sqs.client'; import { ReportingStorage } from '../storage/reporting-storage.service'; import { ReportRepository } from '../repository/report-repository.service'; import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; -import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; +import { ReportCalculator } from '../core/report-calculator.service'; import { CreateReportCalculationUseCase } from '../core/use-cases/create-report-calculation-use-case.service'; -import axios from 'axios'; -import { HttpService } from '@nestjs/axios'; import { ReportSchemaGenerator } from '../core/report-schema-generator'; - -export const ReportCouchSqsClientFactory = ( - configService: ConfigService, -): CouchSqsClient => { - const CONFIG_PREFIX = 'SQS_CLIENT_'; - - const config: CouchSqsClientConfig = { - BASE_URL: configService.getOrThrow(CONFIG_PREFIX + 'BASE_URL'), - BASIC_AUTH_USER: configService.getOrThrow( - CONFIG_PREFIX + 'BASIC_AUTH_USER', - ), - BASIC_AUTH_PASSWORD: configService.getOrThrow( - CONFIG_PREFIX + 'BASIC_AUTH_PASSWORD', - ), - SCHEMA_DESIGN_CONFIG: configService.getOrThrow( - CONFIG_PREFIX + 'SCHEMA_DESIGN_CONFIG', - ), - }; - - const axiosInstance = axios.create(); - - axiosInstance.defaults.baseURL = config.BASE_URL; - axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( - `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, - ).toString('base64')}`; - - return new CouchSqsClient(new HttpService(axiosInstance), config); -}; +import { IReportingStorage } from '../core/report-storage.interface'; +import { IQueryService } from '../../query/core/query-service.interface'; +import { ReportCalculationProcessor } from '../tasks/report-calculation-processor.service'; +import { IReportCalculator } from '../core/report-calculator.interface'; export const ReportingStorageFactory = ( configService: ConfigService, ): ReportingStorage => new ReportingStorage( new ReportRepository( - DefaultCouchDbClientFactory('COUCH_DB_CLIENT_REPORT_', configService), + DefaultCouchDbClientFactory('COUCH_DB_CLIENT_APP_', configService), ), new ReportCalculationRepository( DefaultCouchDbClientFactory( @@ -55,12 +28,17 @@ export const ReportingStorageFactory = ( ); export const SqsReportCalculatorFactory = ( - couchSqsClient: CouchSqsClient, - reportingStorage: ReportingStorage, -): SqsReportCalculator => - new SqsReportCalculator(couchSqsClient, reportingStorage); + queryClient: IQueryService, + reportingStorage: IReportingStorage, +): ReportCalculator => new ReportCalculator(queryClient, reportingStorage); export const CreateReportCalculationUseCaseFactory = ( reportingStorage: ReportingStorage, ): CreateReportCalculationUseCase => new CreateReportCalculationUseCase(reportingStorage); + +export const ReportCalculationProcessorFactory = ( + reportingStorage: IReportingStorage, + reportCalculator: IReportCalculator, +): ReportCalculationProcessor => + new ReportCalculationProcessor(reportingStorage, reportCalculator); diff --git a/src/report/report.module.ts b/src/report/report.module.ts index dae987b..f7c8f9a 100644 --- a/src/report/report.module.ts +++ b/src/report/report.module.ts @@ -5,27 +5,27 @@ import { HttpModule } from '@nestjs/axios'; import { ReportCalculationController } from './controller/report-calculation.controller'; import { ReportCalculationTask } from './tasks/report-calculation-task.service'; import { ReportCalculationProcessor } from './tasks/report-calculation-processor.service'; -import { SqsReportCalculator } from './core/sqs-report-calculator.service'; +import { ReportCalculator } from './core/report-calculator.service'; import { CreateReportCalculationUseCase } from './core/use-cases/create-report-calculation-use-case.service'; -import { CouchSqsClient } from './sqs/couch-sqs.client'; import { ConfigService } from '@nestjs/config'; import { CreateReportCalculationUseCaseFactory, - ReportCouchSqsClientFactory, + ReportCalculationProcessorFactory, ReportingStorageFactory, SqsReportCalculatorFactory, } from './di/report-configuration'; +import { QueryService } from '../query/core/query-service'; +import { QueryModule } from '../query/query.module'; @Module({ controllers: [ReportController, ReportCalculationController], - imports: [HttpModule], + imports: [HttpModule, QueryModule], providers: [ ReportCalculationTask, - ReportCalculationProcessor, { - provide: CouchSqsClient, - useFactory: ReportCouchSqsClientFactory, - inject: [ConfigService], + provide: ReportCalculationProcessor, + useFactory: ReportCalculationProcessorFactory, + inject: [ReportingStorage, ReportCalculator], }, { provide: ReportingStorage, @@ -33,9 +33,9 @@ import { inject: [ConfigService], }, { - provide: SqsReportCalculator, + provide: ReportCalculator, useFactory: SqsReportCalculatorFactory, - inject: [CouchSqsClient, ReportingStorage], + inject: [QueryService, ReportingStorage], }, { provide: CreateReportCalculationUseCase, diff --git a/src/report/sqs/couch-sqs-client.service.spec.ts b/src/report/sqs/couch-sqs-client.service.spec.ts deleted file mode 100644 index c29810a..0000000 --- a/src/report/sqs/couch-sqs-client.service.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { CouchSqsClient } from './couch-sqs.client'; - -describe('CouchSqsClientService', () => { - let service: CouchSqsClient; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [CouchSqsClient], - }).compile(); - - service = module.get(CouchSqsClient); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/report/sqs/couch-sqs.client.ts b/src/report/sqs/couch-sqs.client.ts deleted file mode 100644 index a2c2fb9..0000000 --- a/src/report/sqs/couch-sqs.client.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Logger } from '@nestjs/common'; -import { HttpService } from '@nestjs/axios'; -import { catchError, map, Observable } from 'rxjs'; - -export class CouchSqsClientConfig { - BASE_URL = ''; - BASIC_AUTH_USER = ''; - BASIC_AUTH_PASSWORD = ''; - SCHEMA_DESIGN_CONFIG = ''; -} - -export interface QueryRequest { - query: string; - args?: string[]; -} - -export class CouchSqsClient { - private readonly logger: Logger = new Logger(CouchSqsClient.name); - - constructor( - private httpService: HttpService, - private config: CouchSqsClientConfig, - ) {} - - executeQuery( - query: QueryRequest, - path: string = this.config.SCHEMA_DESIGN_CONFIG, - ): Observable { - return this.httpService.post(path, query).pipe( - map((response) => response.data), - catchError((err) => { - this.logger.error(err); - this.logger.debug( - '[CouchSqsClient] Could not execute Query: ', - this.httpService.axiosRef.defaults.url, - path, - query, - ); - throw err; - }), - ); - } -} diff --git a/src/report/storage/reporting-storage.service.spec.ts b/src/report/storage/reporting-storage.service.spec.ts index ea4484f..95a4eb1 100644 --- a/src/report/storage/reporting-storage.service.spec.ts +++ b/src/report/storage/reporting-storage.service.spec.ts @@ -1,5 +1,5 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { IReportStorage } from '../core/report-storage.interface'; +import { IReportingStorage } from '../core/report-storage.interface'; import { ReportingStorage } from './reporting-storage.service'; import { ReportRepository } from '../repository/report-repository.service'; import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; @@ -8,7 +8,7 @@ import { ConfigService } from '@nestjs/config'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; describe('DefaultReportStorage', () => { - let service: IReportStorage; + let service: IReportingStorage; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -29,7 +29,7 @@ describe('DefaultReportStorage', () => { ], }).compile(); - service = module.get(ReportingStorage); + service = module.get(ReportingStorage); }); it('should be defined', () => { diff --git a/src/report/storage/reporting-storage.service.ts b/src/report/storage/reporting-storage.service.ts index 85ad176..66e12e8 100644 --- a/src/report/storage/reporting-storage.service.ts +++ b/src/report/storage/reporting-storage.service.ts @@ -1,6 +1,6 @@ import { Reference } from '../../domain/reference'; import { Report } from '../../domain/report'; -import { IReportStorage } from '../core/report-storage.interface'; +import { IReportingStorage } from '../core/report-storage.interface'; import { ReportRepository } from '../repository/report-repository.service'; import { map, Observable, Subject, switchMap, tap } from 'rxjs'; import { NotFoundException } from '@nestjs/common'; @@ -15,7 +15,7 @@ import { import { ReportData } from '../../domain/report-data'; import { IReportSchemaGenerator } from '../core/report-schema-generator.interface'; -export class ReportingStorage implements IReportStorage { +export class ReportingStorage implements IReportingStorage { constructor( private reportRepository: ReportRepository, private reportCalculationRepository: ReportCalculationRepository, diff --git a/src/report/tasks/report-calculation-processor.service.spec.ts b/src/report/tasks/report-calculation-processor.service.spec.ts index 738a21f..43bd64a 100644 --- a/src/report/tasks/report-calculation-processor.service.spec.ts +++ b/src/report/tasks/report-calculation-processor.service.spec.ts @@ -1,7 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ReportCalculationProcessor } from './report-calculation-processor.service'; import { ReportingStorage } from '../storage/reporting-storage.service'; -import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; +import { ReportCalculator } from '../core/report-calculator.service'; describe('ReportCalculationProcessorService', () => { let service: ReportCalculationProcessor; @@ -15,7 +15,7 @@ describe('ReportCalculationProcessorService', () => { providers: [ ReportCalculationProcessor, { provide: ReportingStorage, useValue: mockReportStorage }, - { provide: SqsReportCalculator, useValue: mockSqsReportCalculator }, + { provide: ReportCalculator, useValue: mockSqsReportCalculator }, ], }).compile(); diff --git a/src/report/tasks/report-calculation-processor.service.ts b/src/report/tasks/report-calculation-processor.service.ts index 7aa4a6b..c05afd3 100644 --- a/src/report/tasks/report-calculation-processor.service.ts +++ b/src/report/tasks/report-calculation-processor.service.ts @@ -1,20 +1,19 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Logger } from '@nestjs/common'; import { catchError, map, Observable, of, switchMap } from 'rxjs'; import { ReportCalculation, ReportCalculationStatus, } from '../../domain/report-calculation'; -import { ReportingStorage } from '../storage/reporting-storage.service'; -import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; import { ReportData } from '../../domain/report-data'; +import { IReportCalculator } from '../core/report-calculator.interface'; +import { IReportingStorage } from '../core/report-storage.interface'; -@Injectable() export class ReportCalculationProcessor { private readonly logger = new Logger(ReportCalculationProcessor.name); constructor( - private reportStorage: ReportingStorage, - private reportCalculator: SqsReportCalculator, + private reportStorage: IReportingStorage, + private reportCalculator: IReportCalculator, ) {} processNextPendingCalculation(): Observable { diff --git a/src/sql-report.ts b/src/sql-report.ts deleted file mode 100644 index 3329a9c..0000000 --- a/src/sql-report.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * The report entity needs to have the following format in order to work. - * This aligns with the same interface in {@link https://github.com/Aam-Digital/ndb-core} - */ -export interface SqlReport { - mode: 'sql'; - aggregationDefinitions: string[]; -} From da8dbd4e791a03bc45d83fc0df8ffa112c5da0c9 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Tue, 27 Feb 2024 11:21:07 +0100 Subject: [PATCH 02/11] feat: sentry configuration --- package.json | 1 - src/app.module.ts | 55 ++--------------------- src/config/app.yaml | 9 ++++ src/config/configuration.ts | 12 ++++- src/main.ts | 13 ++++-- src/query/di/query.configuration.ts | 8 +++- src/query/domain/QueryRequest.ts | 3 ++ src/query/sqs/sqs.client.spec.ts | 52 ++++++++++------------ src/query/sqs/sqs.client.ts | 3 +- src/sentry.configuration.ts | 68 +++++++++++++++++++++++++++++ tsconfig.json | 2 +- 11 files changed, 134 insertions(+), 92 deletions(-) create mode 100644 src/sentry.configuration.ts diff --git a/package.json b/package.json index cfad1fc..a71b7c2 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,6 @@ "@nestjs/core": "^10.3.3", "@nestjs/platform-express": "^10.3.3", "@nestjs/schedule": "4.0.1", - "@ntegral/nestjs-sentry": "^4.0.1", "@sentry/node": "^7.102.1", "@sentry/tracing": "^7.102.1", "js-yaml": "4.1.0", diff --git a/src/app.module.ts b/src/app.module.ts index 7df9f86..36e1d73 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,32 +1,14 @@ -import { HttpException, Module } from '@nestjs/common'; -import { SentryInterceptor, SentryModule } from '@ntegral/nestjs-sentry'; -import { SeverityLevel } from '@sentry/types'; -import { APP_INTERCEPTOR } from '@nestjs/core'; -import { ConfigModule, ConfigService } from '@nestjs/config'; +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; import { HttpModule } from '@nestjs/axios'; import { ReportModule } from './report/report.module'; import { ScheduleModule } from '@nestjs/schedule'; import { AppConfiguration } from './config/configuration'; import { ReportChangesModule } from './report-changes/report-changes.module'; import { NotificationModule } from './notification/notification.module'; - -const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; +import { QueryModule } from './query/query.module'; @Module({ - providers: [ - { - provide: APP_INTERCEPTOR, - useFactory: () => - new SentryInterceptor({ - filters: [ - { - type: HttpException, - filter: (exception: HttpException) => 500 > exception.getStatus(), // Only report 500 errors - }, - ], - }), - }, - ], imports: [ HttpModule, ScheduleModule.forRoot(), @@ -35,36 +17,7 @@ const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; ignoreEnvFile: false, load: [AppConfiguration], }), - SentryModule.forRootAsync({ - imports: [ConfigModule], - inject: [ConfigService], - useFactory: async (configService: ConfigService) => { - if (!configService.get('SENTRY_DSN')) { - return {}; - } - - return { - dsn: configService.getOrThrow('SENTRY_DSN'), - debug: true, - environment: 'prod', - release: 'backend@' + process.env.npm_package_version, - whitelistUrls: [/https?:\/\/(.*)\.?aam-digital\.com/], - initialScope: { - tags: { - // ID of the docker container in which this is run - hostname: process.env.HOSTNAME || 'unknown', - }, - }, - beforeSend: (event) => { - if (lowSeverityLevels.includes(event.level as SeverityLevel)) { - return null; - } else { - return event; - } - }, - }; - }, - }), + QueryModule, ReportModule, ReportChangesModule, NotificationModule, diff --git a/src/config/app.yaml b/src/config/app.yaml index a5f928e..d7cb93c 100644 --- a/src/config/app.yaml +++ b/src/config/app.yaml @@ -22,3 +22,12 @@ QUERY: TARGET_DATABASE: app REPORT_CHANGES_POLL_INTERVAL: 10000 + +# Logger Configuration +# values can be overwritten in .env file + +SENTRY: + ENABLED: false + INSTANCE_NAME: local-development # can be personalised in .env -> local-development- + ENVIRONMENT: local # local | development | production + DNS: diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 70871a4..fee6e4d 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -4,7 +4,11 @@ import { join } from 'path'; const CONFIG_FILENAME = 'app.yaml'; -export function AppConfiguration() { +/** + * loads local CONFIG_FILENAME file and provides them in NestJs Config Service + * See: /src/config/app.yaml + */ +export function AppConfiguration(): Record { return flatten( yaml.load(readFileSync(join(__dirname, CONFIG_FILENAME), 'utf8')) as Record< string, @@ -16,7 +20,11 @@ export function AppConfiguration() { /** * Recursively create a flat key-value object where keys contain nested keys as prefixes */ -function flatten(obj: any, prefix = '', delimiter = '_') { +function flatten( + obj: any, + prefix = '', + delimiter = '_', +): Record { return Object.keys(obj).reduce((acc: any, k: string) => { const pre = prefix.length ? prefix + delimiter : ''; diff --git a/src/main.ts b/src/main.ts index ca8b82b..01fb3c9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,12 +1,17 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './app.module'; -import { SentryService } from '@ntegral/nestjs-sentry'; +import { ConfigService } from '@nestjs/config'; +import { AppConfiguration } from './config/configuration'; +import { configureSentry } from './sentry.configuration'; +import { INestApplication } from '@nestjs/common'; async function bootstrap() { - const app = await NestFactory.create(AppModule); + // load ConfigService instance to access .env and app.yaml values + const configService = new ConfigService(AppConfiguration()); - // Logging everything through sentry - app.useLogger(SentryService.SentryServiceInstance()); + const app: INestApplication = await NestFactory.create(AppModule); + + configureSentry(app, configService); await app.listen(process.env.PORT || 3000); } diff --git a/src/query/di/query.configuration.ts b/src/query/di/query.configuration.ts index 7bf3b2d..49f5000 100644 --- a/src/query/di/query.configuration.ts +++ b/src/query/di/query.configuration.ts @@ -7,10 +7,12 @@ import { DefaultCouchDbClientFactory } from '../../couchdb/default-factory'; import { EntityConfigResolver } from '../core/entity-config-resolver'; import { QueryService } from '../core/query-service'; import { IEntityConfigResolver } from '../core/entity-config-resolver.interface'; +import { Logger } from '@nestjs/common'; export const SqsClientFactory = ( configService: ConfigService, sqsSchemaService: SqsSchemaService, + logger: Logger, ): SqsClient => { const CONFIG_PREFIX = 'QUERY_SQS_CLIENT_'; @@ -33,7 +35,11 @@ export const SqsClientFactory = ( `${couchSqsClientConfig.BASIC_AUTH_USER}:${couchSqsClientConfig.BASIC_AUTH_PASSWORD}`, ).toString('base64')}`; - return new SqsClient(new HttpService(axiosInstance), sqsSchemaService); + return new SqsClient( + new HttpService(axiosInstance), + sqsSchemaService, + logger, + ); }; export const SqsSchemaServiceFactory = ( diff --git a/src/query/domain/QueryRequest.ts b/src/query/domain/QueryRequest.ts index f810367..0f66286 100644 --- a/src/query/domain/QueryRequest.ts +++ b/src/query/domain/QueryRequest.ts @@ -1,3 +1,6 @@ +/** + * Represent a Query passed into the IQueryService + */ export class QueryRequest { constructor(public query: string) {} } diff --git a/src/query/sqs/sqs.client.spec.ts b/src/query/sqs/sqs.client.spec.ts index f517d2c..143896a 100644 --- a/src/query/sqs/sqs.client.spec.ts +++ b/src/query/sqs/sqs.client.spec.ts @@ -41,9 +41,9 @@ describe('SqsClient', () => { providers: [ { provide: SqsClient, - useFactory: (http, sqsSchemaService) => - new SqsClient(http, sqsSchemaService), - inject: [HttpService, SqsSchemaService], + useFactory: (http, sqsSchemaService, logger) => + new SqsClient(http, sqsSchemaService, logger), + inject: [HttpService, SqsSchemaService, Logger], }, { provide: HttpService, useValue: mockHttp }, { provide: SqsSchemaService, useValue: mockSqsSchemaService }, @@ -59,13 +59,7 @@ describe('SqsClient', () => { }); it('executeQuery() should execute query and return QueryResult', (done) => { - jest - .spyOn(mockSqsSchemaService, 'getSchemaPath') - .mockReturnValue('/app/config_path'); - - jest - .spyOn(mockSqsSchemaService, 'updateSchema') - .mockReturnValue(of(undefined)); + configureSuccessSqsSchemaResponse(); jest.spyOn(mockHttp, 'post').mockReturnValue( of({ @@ -87,8 +81,8 @@ describe('SqsClient', () => { }, }); - expect(mockLogger.error).not.toBeCalled(); - expect(mockLogger.debug).not.toBeCalled(); + expect(mockLogger.error).not.toHaveBeenCalled(); + expect(mockLogger.debug).not.toHaveBeenCalled(); done(); }, @@ -99,13 +93,7 @@ describe('SqsClient', () => { }); it('executeQuery() should handle error from httpService', (done) => { - jest - .spyOn(mockSqsSchemaService, 'getSchemaPath') - .mockReturnValue('/app/config_path'); - - jest - .spyOn(mockSqsSchemaService, 'updateSchema') - .mockReturnValue(of(undefined)); + configureSuccessSqsSchemaResponse(); jest .spyOn(mockHttp, 'post') @@ -116,25 +104,19 @@ describe('SqsClient', () => { query: 'SELECT foo FROM bar', }) .subscribe({ - next: (value) => { + next: () => { done('should throw error '); }, error: (err) => { expect(err.message).toBe('foo error'); - expect(mockLogger.error).not.toBeCalled(); + expect(mockLogger.error).toHaveBeenCalled(); done(); }, }); }); it('executeQueries() should execute all queries and return QueryResult[]', (done) => { - jest - .spyOn(mockSqsSchemaService, 'getSchemaPath') - .mockReturnValue('/app/config_path'); - - jest - .spyOn(mockSqsSchemaService, 'updateSchema') - .mockReturnValue(of(undefined)); + configureSuccessSqsSchemaResponse(); jest.spyOn(mockHttp, 'post').mockReturnValueOnce( of({ @@ -176,8 +158,8 @@ describe('SqsClient', () => { }, ]); - expect(mockLogger.error).not.toBeCalled(); - expect(mockLogger.debug).not.toBeCalled(); + expect(mockLogger.error).not.toHaveBeenCalled(); + expect(mockLogger.debug).not.toHaveBeenCalled(); done(); }, @@ -186,4 +168,14 @@ describe('SqsClient', () => { }, }); }); + + function configureSuccessSqsSchemaResponse() { + jest + .spyOn(mockSqsSchemaService, 'getSchemaPath') + .mockReturnValue('/app/config_path'); + + jest + .spyOn(mockSqsSchemaService, 'updateSchema') + .mockReturnValue(of(undefined)); + } }); diff --git a/src/query/sqs/sqs.client.ts b/src/query/sqs/sqs.client.ts index d07e361..9ec555e 100644 --- a/src/query/sqs/sqs.client.ts +++ b/src/query/sqs/sqs.client.ts @@ -12,11 +12,10 @@ export class CouchSqsClientConfig { } export class SqsClient { - private readonly logger: Logger = new Logger(SqsClient.name); - constructor( private httpService: HttpService, private schemaService: SqsSchemaService, + private logger: Logger, ) {} executeQuery(query: QueryRequest): Observable { diff --git a/src/sentry.configuration.ts b/src/sentry.configuration.ts new file mode 100644 index 0000000..902661a --- /dev/null +++ b/src/sentry.configuration.ts @@ -0,0 +1,68 @@ +import * as Sentry from '@sentry/node'; +import { ConfigService } from '@nestjs/config'; +import { ArgumentsHost, INestApplication } from '@nestjs/common'; +import { BaseExceptionFilter, HttpAdapterHost } from '@nestjs/core'; + +export class SentryConfiguration { + ENABLED = ''; + DNS = ''; + INSTANCE_NAME = ''; + ENVIRONMENT = ''; +} + +function loadSentryConfiguration( + configService: ConfigService, +): SentryConfiguration { + return { + ENABLED: configService.getOrThrow('SENTRY_ENABLED'), + DNS: configService.getOrThrow('SENTRY_DNS'), + INSTANCE_NAME: configService.getOrThrow('SENTRY_INSTANCE_NAME'), + ENVIRONMENT: configService.getOrThrow('SENTRY_ENVIRONMENT'), + }; +} + +export function configureSentry( + app: INestApplication, + configService: ConfigService, +): void { + const sentryConfiguration = loadSentryConfiguration(configService); + if (sentryConfiguration.ENABLED === 'true') { + configureLoggingSentry(app, sentryConfiguration); + } +} + +export class SentryFilter extends BaseExceptionFilter { + catch(exception: unknown, host: ArgumentsHost) { + Sentry.captureException(exception); + super.catch(exception, host); + } +} + +function configureLoggingSentry( + app: INestApplication, + sentryConfiguration: SentryConfiguration, +): void { + Sentry.init({ + debug: true, + serverName: sentryConfiguration.INSTANCE_NAME, + environment: sentryConfiguration.ENVIRONMENT, + dsn: sentryConfiguration.DNS, + integrations: [ + // enable HTTP calls tracing + new Sentry.Integrations.Console(), + new Sentry.Integrations.Http({ tracing: true }), + new Sentry.Integrations.Express(), + ], + // Performance Monitoring + tracesSampleRate: 1.0, // Capture 100% of the transactions + // Set sampling rate for profiling - this is relative to tracesSampleRate + profilesSampleRate: 1.0, + }); + + app.use(Sentry.Handlers.errorHandler()); + app.use(Sentry.Handlers.tracingHandler()); + app.use(Sentry.Handlers.requestHandler()); + + const { httpAdapter } = app.get(HttpAdapterHost); + app.useGlobalFilters(new SentryFilter(httpAdapter)); +} diff --git a/tsconfig.json b/tsconfig.json index 8603582..a52f03c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,7 +6,7 @@ "emitDecoratorMetadata": true, "experimentalDecorators": true, "allowSyntheticDefaultImports": true, - "target": "es2017", + "target": "ES2021", "sourceMap": true, "outDir": "./dist", "baseUrl": "./", From 0d2127b0d700a92629d507f88fecc414b0e36db6 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Tue, 27 Feb 2024 11:26:47 +0100 Subject: [PATCH 03/11] fix: linting --- src/notification/controller/webhook.controller.ts | 10 ++-------- .../storage/webhook-storage.service.ts | 4 +--- src/query/domain/EntityConfig.ts | 15 ++++++++++++--- .../sqs/sqs-schema-generator.service.spec.ts | 1 + 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/notification/controller/webhook.controller.ts b/src/notification/controller/webhook.controller.ts index fcc02d7..52e6c12 100644 --- a/src/notification/controller/webhook.controller.ts +++ b/src/notification/controller/webhook.controller.ts @@ -23,10 +23,7 @@ export class WebhookController { ) {} @Get() - fetchWebhooksOfUser( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - @Headers('Authorization') token: string, - ): Observable { + fetchWebhooksOfUser(): Observable { return this.webhookStorage.fetchAllWebhooks('user-token').pipe( map((webhooks) => webhooks.map((webhook) => this.mapToDto(webhook))), zipAll(), @@ -35,10 +32,7 @@ export class WebhookController { } @Get('/:webhookId') - fetchWebhook( - @Headers('Authorization') token: string, - @Param('webhookId') webhookId: string, - ): Observable { + fetchWebhook(@Param('webhookId') webhookId: string): Observable { return this.webhookStorage.fetchWebhook(new Reference(webhookId)).pipe( // TODO: check auth? map((webhook) => { diff --git a/src/notification/storage/webhook-storage.service.ts b/src/notification/storage/webhook-storage.service.ts index 2283f0d..450f5af 100644 --- a/src/notification/storage/webhook-storage.service.ts +++ b/src/notification/storage/webhook-storage.service.ts @@ -70,10 +70,8 @@ export class WebhookStorage { /** * Get all registered webhooks subscribe by the user authenticated with the given token - * @param token */ - // eslint-disable-next-line @typescript-eslint/no-unused-vars - fetchAllWebhooks(token?: string): Observable { + fetchAllWebhooks(): Observable { return this.webhookRepository .fetchAllWebhooks() .pipe( diff --git a/src/query/domain/EntityConfig.ts b/src/query/domain/EntityConfig.ts index f8ab773..e4de90c 100644 --- a/src/query/domain/EntityConfig.ts +++ b/src/query/domain/EntityConfig.ts @@ -1,11 +1,20 @@ export class EntityAttribute { - constructor(public name: string, public type: string) {} + constructor( + public name: string, + public type: string, + ) {} } export class Entity { - constructor(public label: string, public attributes: EntityAttribute[]) {} + constructor( + public label: string, + public attributes: EntityAttribute[], + ) {} } export class EntityConfig { - constructor(public version: string, public entities: Entity[]) {} + constructor( + public version: string, + public entities: Entity[], + ) {} } diff --git a/src/query/sqs/sqs-schema-generator.service.spec.ts b/src/query/sqs/sqs-schema-generator.service.spec.ts index dcbc349..2c87f89 100644 --- a/src/query/sqs/sqs-schema-generator.service.spec.ts +++ b/src/query/sqs/sqs-schema-generator.service.spec.ts @@ -4,6 +4,7 @@ import { SqsSchemaService } from './sqs-schema-generator.service'; describe('SchemaGeneratorService', () => { let service: SqsSchemaService; + // eslint-disable-next-line @typescript-eslint/no-unused-vars const entityConfig = { 'entity:Child': { label: 'Child', From 574137b0a5646ace19553a9035930a441f9148c5 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Tue, 27 Feb 2024 11:31:32 +0100 Subject: [PATCH 04/11] fix: linting --- src/notification/storage/webhook-storage.service.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/notification/storage/webhook-storage.service.ts b/src/notification/storage/webhook-storage.service.ts index 450f5af..2283f0d 100644 --- a/src/notification/storage/webhook-storage.service.ts +++ b/src/notification/storage/webhook-storage.service.ts @@ -70,8 +70,10 @@ export class WebhookStorage { /** * Get all registered webhooks subscribe by the user authenticated with the given token + * @param token */ - fetchAllWebhooks(): Observable { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + fetchAllWebhooks(token?: string): Observable { return this.webhookRepository .fetchAllWebhooks() .pipe( From 2a69952e969ae8e7fe9367eb6516c532a19996aa Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Wed, 28 Feb 2024 10:30:47 +0100 Subject: [PATCH 05/11] chore: npm install --- package-lock.json | 405 +++++----------------------------------------- 1 file changed, 39 insertions(+), 366 deletions(-) diff --git a/package-lock.json b/package-lock.json index 42d9a9d..b1f8b0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,6 @@ "@nestjs/core": "^10.3.3", "@nestjs/platform-express": "^10.3.3", "@nestjs/schedule": "4.0.1", - "@ntegral/nestjs-sentry": "^4.0.1", "@sentry/node": "^7.102.1", "@sentry/tracing": "^7.102.1", "js-yaml": "4.1.0", @@ -1028,67 +1027,6 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/@graphql-tools/merge": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@graphql-tools/merge/-/merge-9.0.0.tgz", - "integrity": "sha512-J7/xqjkGTTwOJmaJQJ2C+VDBDOWJL3lKrHJN4yMaRLAJH3PosB7GiPRaSDZdErs0+F77sH2MKs2haMMkywzx7Q==", - "optional": true, - "dependencies": { - "@graphql-tools/utils": "^10.0.0", - "tslib": "^2.4.0" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" - } - }, - "node_modules/@graphql-tools/schema": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/@graphql-tools/schema/-/schema-10.0.0.tgz", - "integrity": "sha512-kf3qOXMFcMs2f/S8Y3A8fm/2w+GaHAkfr3Gnhh2LOug/JgpY/ywgFVxO3jOeSpSEdoYcDKLcXVjMigNbY4AdQg==", - "optional": true, - "dependencies": { - "@graphql-tools/merge": "^9.0.0", - "@graphql-tools/utils": "^10.0.0", - "tslib": "^2.4.0", - "value-or-promise": "^1.0.12" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" - } - }, - "node_modules/@graphql-tools/utils": { - "version": "10.0.8", - "resolved": "https://registry.npmjs.org/@graphql-tools/utils/-/utils-10.0.8.tgz", - "integrity": "sha512-yjyA8ycSa1WRlJqyX/aLqXeE5DvF/H02+zXMUFnCzIDrj0UvLMUrxhmVFnMK0Q2n3bh4uuTeY3621m5za9ovXw==", - "optional": true, - "dependencies": { - "@graphql-typed-document-node/core": "^3.1.1", - "cross-inspect": "1.0.0", - "dset": "^3.1.2", - "tslib": "^2.4.0" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" - } - }, - "node_modules/@graphql-typed-document-node/core": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@graphql-typed-document-node/core/-/core-3.2.0.tgz", - "integrity": "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==", - "optional": true, - "peerDependencies": { - "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -1926,99 +1864,6 @@ } } }, - "node_modules/@nestjs/graphql": { - "version": "12.0.11", - "resolved": "https://registry.npmjs.org/@nestjs/graphql/-/graphql-12.0.11.tgz", - "integrity": "sha512-iCyVs9+utCQt9ehMhUjQcEdjRN/MrcTBINd7P44O1fzGENuWMbt1Z8RCoZbeGi5iVPBY63HgYik+BnnICqmxZw==", - "optional": true, - "dependencies": { - "@graphql-tools/merge": "9.0.0", - "@graphql-tools/schema": "10.0.0", - "@graphql-tools/utils": "10.0.8", - "@nestjs/mapped-types": "2.0.2", - "chokidar": "3.5.3", - "fast-glob": "3.3.2", - "graphql-tag": "2.12.6", - "graphql-ws": "5.14.2", - "lodash": "4.17.21", - "normalize-path": "3.0.0", - "subscriptions-transport-ws": "0.11.0", - "tslib": "2.6.2", - "uuid": "9.0.1", - "ws": "8.14.2" - }, - "peerDependencies": { - "@apollo/subgraph": "^2.0.0", - "@nestjs/common": "^9.3.8 || ^10.0.0", - "@nestjs/core": "^9.3.8 || ^10.0.0", - "class-transformer": "*", - "class-validator": "*", - "graphql": "^16.6.0", - "reflect-metadata": "^0.1.13", - "ts-morph": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^20.0.0" - }, - "peerDependenciesMeta": { - "@apollo/subgraph": { - "optional": true - }, - "class-transformer": { - "optional": true - }, - "class-validator": { - "optional": true - }, - "ts-morph": { - "optional": true - } - } - }, - "node_modules/@nestjs/graphql/node_modules/chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "optional": true, - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/@nestjs/mapped-types": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@nestjs/mapped-types/-/mapped-types-2.0.2.tgz", - "integrity": "sha512-V0izw6tWs6fTp9+KiiPUbGHWALy563Frn8X6Bm87ANLRuE46iuBMD5acKBDP5lKL/75QFvrzSJT7HkCbB0jTpg==", - "optional": true, - "peerDependencies": { - "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", - "class-transformer": "^0.4.0 || ^0.5.0", - "class-validator": "^0.13.0 || ^0.14.0", - "reflect-metadata": "^0.1.12" - }, - "peerDependenciesMeta": { - "class-transformer": { - "optional": true - }, - "class-validator": { - "optional": true - } - } - }, "node_modules/@nestjs/platform-express": { "version": "10.3.3", "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.3.tgz", @@ -2105,7 +1950,7 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "devOptional": true, + "dev": true, "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -2118,7 +1963,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "devOptional": true, + "dev": true, "engines": { "node": ">= 8" } @@ -2127,7 +1972,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "devOptional": true, + "dev": true, "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -2136,23 +1981,6 @@ "node": ">= 8" } }, - "node_modules/@ntegral/nestjs-sentry": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@ntegral/nestjs-sentry/-/nestjs-sentry-4.0.1.tgz", - "integrity": "sha512-GQUL0Bm0T+FhTNJXUbnF5mZc2u5YuvUV2H6naXxrnw8tY0b9eE/DGj+GUyHNL7V2DuHHFzsYP2c30O5FoGoYfQ==", - "optionalDependencies": { - "@nestjs/graphql": "~12.0.11" - }, - "peerDependencies": { - "@nestjs/common": ">=10.0.0", - "@nestjs/core": ">=10.0.0", - "@sentry/hub": "^7.7.0", - "@sentry/node": "^7.7.0", - "reflect-metadata": "^0.1.13", - "rimraf": "^3.0.2", - "rxjs": "^7.2.0" - } - }, "node_modules/@nuxtjs/opencollective": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@nuxtjs/opencollective/-/opencollective-0.3.2.tgz", @@ -2217,20 +2045,6 @@ "node": ">=8" } }, - "node_modules/@sentry/hub": { - "version": "7.103.0", - "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-7.103.0.tgz", - "integrity": "sha512-61o906jj5i2T9iUeFy/UwisLaCYt5SMU6i64w3pbFM4CJJRsufc+zcrSDfngLJMYOawlXJPa85iekBEkfpOzzQ==", - "peer": true, - "dependencies": { - "@sentry/core": "7.103.0", - "@sentry/types": "7.103.0", - "@sentry/utils": "7.103.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@sentry/node": { "version": "7.103.0", "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.103.0.tgz", @@ -3138,7 +2952,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "devOptional": true, + "dev": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -3321,16 +3135,11 @@ "@babel/core": "^7.0.0" } }, - "node_modules/backo2": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", - "integrity": "sha512-zj6Z6M7Eq+PBZ7PQxl5NT665MvJdAkzp0f60nAJ+sLaSCBPMwVak5ZegFbgVCzFcCJTKFoMizvM5Ld7+JrRJHA==", - "optional": true - }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true }, "node_modules/base64-js": { "version": "1.5.1", @@ -3356,7 +3165,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "devOptional": true, + "dev": true, "engines": { "node": ">=8" } @@ -3426,6 +3235,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -3435,7 +3245,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "devOptional": true, + "dev": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -3827,7 +3637,8 @@ "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true }, "node_modules/concat-stream": { "version": "1.6.2", @@ -3971,18 +3782,6 @@ "luxon": "~3.4.0" } }, - "node_modules/cross-inspect": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/cross-inspect/-/cross-inspect-1.0.0.tgz", - "integrity": "sha512-4PFfn4b5ZN6FMNGSZlyb7wUhuN8wvj8t/VQHZdM4JsDcruGJ8L2kf9zao98QIrBPFCpdk27qst/AGTl7pL3ypQ==", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -4173,15 +3972,6 @@ "node": ">=12" } }, - "node_modules/dset": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.3.tgz", - "integrity": "sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ==", - "optional": true, - "engines": { - "node": ">=4" - } - }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -4558,12 +4348,6 @@ "node": ">= 0.6" } }, - "node_modules/eventemitter3": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", - "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==", - "optional": true - }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -4747,7 +4531,7 @@ "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "devOptional": true, + "dev": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -4780,7 +4564,7 @@ "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", - "devOptional": true, + "dev": true, "dependencies": { "reusify": "^1.0.4" } @@ -4834,7 +4618,7 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "devOptional": true, + "dev": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -5051,12 +4835,14 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -5131,6 +4917,7 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -5150,7 +4937,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "devOptional": true, + "dev": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -5222,43 +5009,6 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, - "node_modules/graphql": { - "version": "16.8.1", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", - "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", - "optional": true, - "peer": true, - "engines": { - "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" - } - }, - "node_modules/graphql-tag": { - "version": "2.12.6", - "resolved": "https://registry.npmjs.org/graphql-tag/-/graphql-tag-2.12.6.tgz", - "integrity": "sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg==", - "optional": true, - "dependencies": { - "tslib": "^2.1.0" - }, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "graphql": "^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0" - } - }, - "node_modules/graphql-ws": { - "version": "5.14.2", - "resolved": "https://registry.npmjs.org/graphql-ws/-/graphql-ws-5.14.2.tgz", - "integrity": "sha512-LycmCwhZ+Op2GlHz4BZDsUYHKRiiUz+3r9wbhBATMETNlORQJAaFlAgTFoeRh6xQoQegwYwIylVD1Qns9/DA3w==", - "optional": true, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "graphql": ">=0.11 <=16" - } - }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -5447,6 +5197,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -5524,7 +5275,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "devOptional": true, + "dev": true, "dependencies": { "binary-extensions": "^2.0.0" }, @@ -5548,7 +5299,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "devOptional": true, + "dev": true, "engines": { "node": ">=0.10.0" } @@ -5575,7 +5326,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "devOptional": true, + "dev": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -5596,7 +5347,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "devOptional": true, + "dev": true, "engines": { "node": ">=0.12.0" } @@ -5720,12 +5471,6 @@ "node": ">=8" } }, - "node_modules/iterall": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/iterall/-/iterall-1.3.0.tgz", - "integrity": "sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg==", - "optional": true - }, "node_modules/iterare": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/iterare/-/iterare-1.2.1.tgz", @@ -6606,7 +6351,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "devOptional": true, + "dev": true, "engines": { "node": ">= 8" } @@ -6623,7 +6368,7 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "devOptional": true, + "dev": true, "dependencies": { "braces": "^3.0.2", "picomatch": "^2.3.1" @@ -6675,6 +6420,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -6809,7 +6555,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "devOptional": true, + "dev": true, "engines": { "node": ">=0.10.0" } @@ -6857,6 +6603,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, "dependencies": { "wrappy": "1" } @@ -7015,6 +6762,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -7083,7 +6831,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "devOptional": true, + "dev": true, "engines": { "node": ">=8.6" }, @@ -7314,7 +7062,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "devOptional": true, + "dev": true, "funding": [ { "type": "github", @@ -7390,7 +7138,7 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "devOptional": true, + "dev": true, "dependencies": { "picomatch": "^2.2.1" }, @@ -7515,7 +7263,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "devOptional": true, + "dev": true, "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" @@ -7525,6 +7273,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, "dependencies": { "glob": "^7.1.3" }, @@ -7548,7 +7297,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "devOptional": true, + "dev": true, "funding": [ { "type": "github", @@ -8017,53 +7766,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/subscriptions-transport-ws": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/subscriptions-transport-ws/-/subscriptions-transport-ws-0.11.0.tgz", - "integrity": "sha512-8D4C6DIH5tGiAIpp5I0wD/xRlNiZAPGHygzCe7VzyzUoxHtawzjNAY9SUTXU05/EY2NMY9/9GF0ycizkXr1CWQ==", - "deprecated": "The `subscriptions-transport-ws` package is no longer maintained. We recommend you use `graphql-ws` instead. For help migrating Apollo software to `graphql-ws`, see https://www.apollographql.com/docs/apollo-server/data/subscriptions/#switching-from-subscriptions-transport-ws For general help using `graphql-ws`, see https://github.com/enisdenjo/graphql-ws/blob/master/README.md", - "optional": true, - "dependencies": { - "backo2": "^1.0.2", - "eventemitter3": "^3.1.0", - "iterall": "^1.2.1", - "symbol-observable": "^1.0.4", - "ws": "^5.2.0 || ^6.0.0 || ^7.0.0" - }, - "peerDependencies": { - "graphql": "^15.7.2 || ^16.0.0" - } - }, - "node_modules/subscriptions-transport-ws/node_modules/symbol-observable": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", - "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/subscriptions-transport-ws/node_modules/ws": { - "version": "7.5.9", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", - "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", - "optional": true, - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/superagent": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/superagent/-/superagent-8.1.2.tgz", @@ -8311,7 +8013,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "devOptional": true, + "dev": true, "dependencies": { "is-number": "^7.0.0" }, @@ -8682,15 +8384,6 @@ "node": ">=10.12.0" } }, - "node_modules/value-or-promise": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/value-or-promise/-/value-or-promise-1.0.12.tgz", - "integrity": "sha512-Z6Uz+TYwEqE7ZN50gwn+1LCVo9ZVrpxRPOhOLnncYkY1ZzOYtrX8Fwf/rFktZ8R5mJms6EZf5TqNOMeZmnPq9Q==", - "optional": true, - "engines": { - "node": ">=12" - } - }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -8862,7 +8555,8 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true }, "node_modules/write-file-atomic": { "version": "4.0.2", @@ -8877,27 +8571,6 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, - "node_modules/ws": { - "version": "8.14.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", - "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", - "optional": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", From a4489d72fcc1074823a7aeed8e1820bafb56b437 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Wed, 28 Feb 2024 14:11:17 +0100 Subject: [PATCH 06/11] test: add test cases --- src/config/app.yaml | 2 +- src/couchdb/couch-db-client.interface.ts | 25 ++ src/couchdb/couch-db-client.service.ts | 10 +- src/query/sqs/dtos.ts | 42 ++- .../sqs/sqs-schema-generator.service.spec.ts | 315 ++++++++++++------ src/query/sqs/sqs-schema-generator.service.ts | 77 +++-- 6 files changed, 329 insertions(+), 142 deletions(-) create mode 100644 src/couchdb/couch-db-client.interface.ts diff --git a/src/config/app.yaml b/src/config/app.yaml index d7cb93c..8d7f34c 100644 --- a/src/config/app.yaml +++ b/src/config/app.yaml @@ -30,4 +30,4 @@ SENTRY: ENABLED: false INSTANCE_NAME: local-development # can be personalised in .env -> local-development- ENVIRONMENT: local # local | development | production - DNS: + DNS: '' diff --git a/src/couchdb/couch-db-client.interface.ts b/src/couchdb/couch-db-client.interface.ts new file mode 100644 index 0000000..f8fb080 --- /dev/null +++ b/src/couchdb/couch-db-client.interface.ts @@ -0,0 +1,25 @@ +import { Observable } from 'rxjs'; +import { CouchDbChangesResponse } from './dtos'; +import { AxiosResponse } from 'axios'; + +export interface ICouchDbClient { + changes(request: { config?: any }): Observable; + + headDatabaseDocument(request: { + documentId: string; + config?: any; + }): Observable>; + + getDatabaseDocument(request: { + documentId: string; + config?: any; + }): Observable; + + find(request: { query: object; config: any }): Observable; + + putDatabaseDocument(request: { + documentId: string; + body: any; + config: any; + }): Observable; +} diff --git a/src/couchdb/couch-db-client.service.ts b/src/couchdb/couch-db-client.service.ts index fbd226f..8706c82 100644 --- a/src/couchdb/couch-db-client.service.ts +++ b/src/couchdb/couch-db-client.service.ts @@ -1,8 +1,9 @@ import { Logger } from '@nestjs/common'; import { catchError, map, Observable, of, switchMap } from 'rxjs'; import { HttpService } from '@nestjs/axios'; -import { AxiosHeaders } from 'axios'; +import { AxiosHeaders, AxiosResponse } from 'axios'; import { CouchDbChangesResponse } from './dtos'; +import { ICouchDbClient } from './couch-db-client.interface'; export class CouchDbClientConfig { BASE_URL = ''; @@ -11,7 +12,7 @@ export class CouchDbClientConfig { BASIC_AUTH_PASSWORD = ''; } -export class CouchDbClient { +export class CouchDbClient implements ICouchDbClient { private readonly logger = new Logger(CouchDbClient.name); constructor(private httpService: HttpService) {} @@ -30,7 +31,10 @@ export class CouchDbClient { ); } - headDatabaseDocument(request: { documentId: string; config?: any }) { + headDatabaseDocument(request: { + documentId: string; + config?: any; + }): Observable> { return this.httpService.head(`${request.documentId}`, request.config).pipe( catchError((err) => { if (err.response.status !== 404) { diff --git a/src/query/sqs/dtos.ts b/src/query/sqs/dtos.ts index ad52b82..ee6268a 100644 --- a/src/query/sqs/dtos.ts +++ b/src/query/sqs/dtos.ts @@ -1,15 +1,37 @@ +import * as crypto from 'crypto'; + export class SqsSchema { + readonly language: 'sqlite'; + readonly configVersion: string; + readonly sql: { + tables: SqlTables; + // Optional SQL indices + indexes: string[]; + // Further options + options: SqlOptions; + }; + constructor( - public sql: { - // SQL table definitions - tables: SqlTables; - // Optional SQL indices - indexes?: string[]; - // Further options - options?: SqlOptions; - }, - public language: 'sqlite' = 'sqlite', - ) {} + tables: SqlTables, + indexes: string[], + options: SqlOptions, + language: 'sqlite' = 'sqlite', + ) { + this.sql = { + tables: tables, + indexes: indexes, + options: options, + }; + this.language = language; + this.configVersion = this.asHash(); + } + + private asHash(): string { + return crypto + .createHash('sha256') + .update(JSON.stringify(this.sql)) + .digest('hex'); + } } type SqlTables = { diff --git a/src/query/sqs/sqs-schema-generator.service.spec.ts b/src/query/sqs/sqs-schema-generator.service.spec.ts index 2c87f89..cea79d9 100644 --- a/src/query/sqs/sqs-schema-generator.service.spec.ts +++ b/src/query/sqs/sqs-schema-generator.service.spec.ts @@ -1,122 +1,138 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { SqsSchemaService } from './sqs-schema-generator.service'; +import { + SqsSchemaGeneratorConfig, + SqsSchemaService, +} from './sqs-schema-generator.service'; +import { of, throwError } from 'rxjs'; +import { EntityConfig } from '../domain/EntityConfig'; +import { DocSuccess } from '../../couchdb/dtos'; +import spyOn = jest.spyOn; describe('SchemaGeneratorService', () => { let service: SqsSchemaService; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const entityConfig = { - 'entity:Child': { - label: 'Child', - labelPlural: 'Children', - attributes: { - address: { - dataType: 'location', - label: 'Address', - }, - health_bloodGroup: { - dataType: 'string', - label: 'Blood Group', - }, - religion: { - dataType: 'string', - label: 'Religion', - }, - motherTongue: { - dataType: 'string', - label: 'Mother Tongue', - description: 'The primary language spoken at home', - }, - health_lastDentalCheckup: { - dataType: 'date', - label: 'Last Dental Check-Up', - }, - birth_certificate: { - dataType: 'file', - label: 'Birth certificate', - }, + let mockCouchDbClient: { + changes: jest.Mock; + headDatabaseDocument: jest.Mock; + getDatabaseDocument: jest.Mock; + find: jest.Mock; + putDatabaseDocument: jest.Mock; + }; + + let mockEntityConfigResolver: { + getEntityConfig: jest.Mock; + }; + + const sqsSchemaGeneratorConfig: SqsSchemaGeneratorConfig = { + SCHEMA_PATH: '/_design/sqlite:config', + }; + + const entityConfig: EntityConfig = { + version: 'rev-1', + entities: [ + { + label: 'Child', + attributes: [ + { + name: 'name', + type: 'TEXT', + }, + { + name: 'age', + type: 'INTEGER', + }, + ], }, - }, - 'entity:School': { - attributes: { - name: { - dataType: 'string', - label: 'Name', - }, - privateSchool: { - dataType: 'boolean', - label: 'Private School', - }, - language: { - dataType: 'string', - label: 'Language', - }, - address: { - dataType: 'location', - label: 'Address', - }, - phone: { - dataType: 'string', - label: 'Phone Number', - }, - timing: { - dataType: 'string', - label: 'School Timing', - }, - remarks: { - dataType: 'string', - label: 'Remarks', - }, + { + label: 'School', + attributes: [ + { + name: 'name', + type: 'TEXT', + }, + { + name: 'type', + type: 'TEXT', + }, + { + name: 'numberOfStudents', + type: 'INTEGER', + }, + ], }, - }, - 'entity:HistoricalEntityData': { - attributes: { - isMotivatedDuringClass: { - dataType: 'configurable-enum', - additional: 'rating-answer', - label: 'Motivated', - description: 'The child is motivated during the class.', - }, - isParticipatingInClass: { - dataType: 'configurable-enum', - additional: 'rating-answer', - label: 'Participating', - description: 'The child is actively participating in the class.', - }, - isInteractingWithOthers: { - dataType: 'configurable-enum', - additional: 'rating-answer', - label: 'Interacting', - description: - 'The child interacts with other students during the class.', - }, - doesHomework: { - dataType: 'configurable-enum', - additional: 'rating-answer', - label: 'Homework', - description: 'The child does its homework.', - }, - asksQuestions: { - dataType: 'configurable-enum', - additional: 'rating-answer', - label: 'Asking Questions', - description: 'The child is asking questions during the class.', + ], + }; + + const sqsConfig = { + _id: '_design/sqlite:config', + _rev: '1-00000000', + sql: { + tables: { + Child: { + fields: { + name: 'TEXT', + age: 'INTEGER', + _id: 'TEXT', + _rev: 'TEXT', + created: 'TEXT', + updated: 'TEXT', + inactive: 'INTEGER', + anonymized: 'INTEGER', + }, + }, + School: { + fields: { + name: 'TEXT', + type: 'TEXT', + numberOfStudents: 'INTEGER', + _id: 'TEXT', + _rev: 'TEXT', + created: 'TEXT', + updated: 'TEXT', + inactive: 'INTEGER', + anonymized: 'INTEGER', + }, }, }, - }, - 'entity:User': { - attributes: { - phone: { - dataType: 'string', - label: 'Contact', + indexes: [], + options: { + table_name: { + operation: 'prefix', + field: '_id', + separator: ':', }, }, }, + language: 'sqlite', + configVersion: + '2a26f7bc7e7e69940d811a4845a5f88374cbbb9868c8f4ce13303c3be71f2ad8', }; beforeEach(async () => { + mockCouchDbClient = { + changes: jest.fn(), + headDatabaseDocument: jest.fn(), + getDatabaseDocument: jest.fn(), + find: jest.fn(), + putDatabaseDocument: jest.fn(), + }; + + mockEntityConfigResolver = { + getEntityConfig: jest.fn(), + }; + const module: TestingModule = await Test.createTestingModule({ - providers: [SqsSchemaService], + providers: [ + { + provide: SqsSchemaService, + useFactory: () => + new SqsSchemaService( + mockCouchDbClient, + mockEntityConfigResolver, + sqsSchemaGeneratorConfig, + ), + }, + ], }).compile(); service = module.get(SqsSchemaService); @@ -125,4 +141,97 @@ describe('SchemaGeneratorService', () => { it('should be defined', () => { expect(service).toBeDefined(); }); + + it('getSchemaPath() should return Schema', () => { + expect(service.getSchemaPath()).toEqual('/_design/sqlite:config'); + }); + + it('updateSchema() should update Schema when new sqs config version is different then current', (done) => { + // given + resolveEntityConfig(); + resolveSqsConfigWithOtherVersion(); + resolveDocSuccess(); + // when + service.updateSchema().subscribe({ + next: () => { + // then + expect(mockCouchDbClient.putDatabaseDocument).toHaveBeenCalled(); + done(); + }, + error: (err) => { + done(err); + }, + }); + }); + + it('updateSchema() should update Schema when no sqs could be fetched', (done) => { + // given + resolveEntityConfig(); + resolveSqsConfigNotFound(); + resolveDocSuccess(); + // when + service.updateSchema().subscribe({ + next: () => { + // then + expect(mockCouchDbClient.putDatabaseDocument).toHaveBeenCalled(); + done(); + }, + error: (err) => { + done(err); + }, + }); + }); + + it('updateSchema() should not update Schema when entity:config is unchanged', (done) => { + // given + resolveEntityConfig(); + resolveSqsConfig(); + // when + service.updateSchema().subscribe({ + next: () => { + // then + expect(mockCouchDbClient.putDatabaseDocument).not.toHaveBeenCalled(); + done(); + }, + error: (err) => { + done(err); + }, + }); + }); + + function resolveEntityConfig() { + spyOn(mockEntityConfigResolver, 'getEntityConfig').mockReturnValue( + of(entityConfig), + ); + } + + function resolveSqsConfig() { + spyOn(mockCouchDbClient, 'getDatabaseDocument').mockReturnValue( + of(sqsConfig), + ); + } + + function resolveDocSuccess() { + spyOn(mockCouchDbClient, 'putDatabaseDocument').mockReturnValue( + of(new DocSuccess(true, 'id-123', 'r-123')), + ); + } + + function resolveSqsConfigWithOtherVersion() { + const sqsConfigWithNewVersion = { ...sqsConfig }; + sqsConfigWithNewVersion.configVersion = '123'; + spyOn(mockCouchDbClient, 'getDatabaseDocument').mockReturnValue( + of(sqsConfigWithNewVersion), + ); + } + + function resolveSqsConfigNotFound() { + const sqsConfigWithNewVersion = { ...sqsConfig }; + sqsConfigWithNewVersion.configVersion = '123'; + spyOn(mockCouchDbClient, 'getDatabaseDocument').mockReturnValue( + throwError(() => { + throw new Error('not found'); + }), + ); + } }); diff --git a/src/query/sqs/sqs-schema-generator.service.ts b/src/query/sqs/sqs-schema-generator.service.ts index ecf3bc2..275b112 100644 --- a/src/query/sqs/sqs-schema-generator.service.ts +++ b/src/query/sqs/sqs-schema-generator.service.ts @@ -1,19 +1,17 @@ -import { EMPTY, map, Observable, switchMap } from 'rxjs'; -import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { catchError, Observable, of, switchMap, tap, zipWith } from 'rxjs'; import { IEntityConfigResolver } from '../core/entity-config-resolver.interface'; import { SqsSchema } from './dtos'; import { DocSuccess } from '../../couchdb/dtos'; import { EntityAttribute, EntityConfig } from '../domain/EntityConfig'; +import { ICouchDbClient } from '../../couchdb/couch-db-client.interface'; export class SqsSchemaGeneratorConfig { SCHEMA_PATH = ''; } export class SqsSchemaService { - private _entityConfigVersion = ''; - constructor( - private couchDbClient: CouchDbClient, + private couchDbClient: ICouchDbClient, private entityConfigResolver: IEntityConfigResolver, private config: SqsSchemaGeneratorConfig, ) {} @@ -23,32 +21,57 @@ export class SqsSchemaService { } /** - * Loads EntityConfig and updates SqsSchema if necessary + * Loads EntityConfig and current SQS Schema. Updates SqsSchema if necessary */ updateSchema(): Observable { return this.entityConfigResolver.getEntityConfig().pipe( - switchMap((entityConfig) => { - if (entityConfig.version === this._entityConfigVersion) { - return EMPTY; - } else { - return this.couchDbClient - .putDatabaseDocument({ - documentId: this.config.SCHEMA_PATH, - body: this.mapToSqsSchema(entityConfig), - config: {}, - }) - .pipe( - map((result) => { - this._entityConfigVersion = result.rev; - }), - ); + zipWith( + this.couchDbClient + .getDatabaseDocument({ + documentId: this.config.SCHEMA_PATH, + }) + .pipe( + catchError(() => { + console.debug( + '[SqsSchemaService] No active sqs schema found in db.', + ); + return of(undefined); + }), + ), + ), + switchMap((result) => { + const entityConfig = result[0]; + const currentSqsSchema = result[1]; + const newSqsSchema = this.mapToSqsSchema(entityConfig); + + if (currentSqsSchema?.configVersion === newSqsSchema.configVersion) { + console.debug( + '[SqsSchemaService] sqs schema is up to date. not updated.', + ); + return of(undefined); } + + return this.couchDbClient + .putDatabaseDocument({ + documentId: this.config.SCHEMA_PATH, + body: newSqsSchema, + config: {}, + }) + .pipe( + tap((result) => { + console.debug( + '[SqsSchemaService] sqs schema updated to latest version', + result, + ); + }), + switchMap(() => of(undefined)), + ); }), ); } private mapToSqsSchema(entityConfig: EntityConfig): SqsSchema { - const sqsSchema = new SqsSchema({ + const sqsSchema: any = { tables: {}, indexes: [], options: { @@ -58,7 +81,7 @@ export class SqsSchemaService { separator: ':', }, }, - }); + }; entityConfig.entities.forEach((entityConfig) => { const fields: { @@ -77,12 +100,16 @@ export class SqsSchemaService { } }); - sqsSchema.sql.tables[entityConfig.label] = { + sqsSchema.tables[entityConfig.label] = { fields: fields, }; }); - return sqsSchema; + return new SqsSchema( + sqsSchema.tables, + sqsSchema.indexes, + sqsSchema.options, + ); } private getDefaultEntityAttributes(): EntityAttribute[] { From 721a53f28574bdad8f15ad65c9c3cae851e91fb7 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Thu, 29 Feb 2024 10:58:05 +0100 Subject: [PATCH 07/11] chore: remove merge relicts --- src/app.module.ts | 55 +++-------------------------------------------- 1 file changed, 3 insertions(+), 52 deletions(-) diff --git a/src/app.module.ts b/src/app.module.ts index e333e93..6c415e4 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,8 +1,5 @@ -import { HttpException, Module } from '@nestjs/common'; -import { SentryInterceptor, SentryModule } from '@ntegral/nestjs-sentry'; -import { SeverityLevel } from '@sentry/types'; -import { APP_INTERCEPTOR } from '@nestjs/core'; -import { ConfigModule, ConfigService } from '@nestjs/config'; +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; import { HttpModule } from '@nestjs/axios'; import { ReportModule } from './report/report.module'; import { ScheduleModule } from '@nestjs/schedule'; @@ -12,23 +9,7 @@ import { NotificationModule } from './notification/notification.module'; import { QueryModule } from './query/query.module'; import { AuthModule } from './auth/auth.module'; -const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; - @Module({ - providers: [ - { - provide: APP_INTERCEPTOR, - useFactory: () => - new SentryInterceptor({ - filters: [ - { - type: HttpException, - filter: (exception: HttpException) => 500 > exception.getStatus(), // Only report 500 errors - }, - ], - }), - }, - ], imports: [ HttpModule, ScheduleModule.forRoot(), @@ -37,38 +18,8 @@ const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; ignoreEnvFile: false, load: [AppConfiguration], }), - QueryModule, AuthModule, - SentryModule.forRootAsync({ - imports: [ConfigModule], - inject: [ConfigService], - useFactory: async (configService: ConfigService) => { - if (!configService.get('SENTRY_DSN')) { - return {}; - } - - return { - dsn: configService.getOrThrow('SENTRY_DSN'), - debug: true, - environment: 'prod', - release: 'backend@' + process.env.npm_package_version, - whitelistUrls: [/https?:\/\/(.*)\.?aam-digital\.com/], - initialScope: { - tags: { - // ID of the docker container in which this is run - hostname: process.env.HOSTNAME || 'unknown', - }, - }, - beforeSend: (event) => { - if (lowSeverityLevels.includes(event.level as SeverityLevel)) { - return null; - } else { - return event; - } - }, - }; - }, - }), + QueryModule, ReportModule, ReportChangesModule, NotificationModule, From 076a5b32aa52e99c8402a4da9cbbfbdd2f68d3b8 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Thu, 29 Feb 2024 11:01:25 +0100 Subject: [PATCH 08/11] Update src/query/sqs/sqs-schema-generator.service.spec.ts Co-authored-by: Sebastian --- src/query/sqs/sqs-schema-generator.service.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/sqs/sqs-schema-generator.service.spec.ts b/src/query/sqs/sqs-schema-generator.service.spec.ts index cea79d9..7310e9a 100644 --- a/src/query/sqs/sqs-schema-generator.service.spec.ts +++ b/src/query/sqs/sqs-schema-generator.service.spec.ts @@ -8,7 +8,7 @@ import { EntityConfig } from '../domain/EntityConfig'; import { DocSuccess } from '../../couchdb/dtos'; import spyOn = jest.spyOn; -describe('SchemaGeneratorService', () => { +describe('SqsSchemaService', () => { let service: SqsSchemaService; let mockCouchDbClient: { From ad1c2de36c57544a2e39e6efef5c51ec23d7fb72 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Thu, 29 Feb 2024 11:02:04 +0100 Subject: [PATCH 09/11] fix: rename sentry config property from DNS to DSN --- src/config/app.yaml | 2 +- src/sentry.configuration.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config/app.yaml b/src/config/app.yaml index cebdb23..3072f5a 100644 --- a/src/config/app.yaml +++ b/src/config/app.yaml @@ -32,4 +32,4 @@ SENTRY: ENABLED: false INSTANCE_NAME: local-development # can be personalised in .env -> local-development- ENVIRONMENT: local # local | development | production - DNS: '' + DSN: '' diff --git a/src/sentry.configuration.ts b/src/sentry.configuration.ts index 902661a..fcbcc5b 100644 --- a/src/sentry.configuration.ts +++ b/src/sentry.configuration.ts @@ -5,7 +5,7 @@ import { BaseExceptionFilter, HttpAdapterHost } from '@nestjs/core'; export class SentryConfiguration { ENABLED = ''; - DNS = ''; + DSN = ''; INSTANCE_NAME = ''; ENVIRONMENT = ''; } @@ -15,7 +15,7 @@ function loadSentryConfiguration( ): SentryConfiguration { return { ENABLED: configService.getOrThrow('SENTRY_ENABLED'), - DNS: configService.getOrThrow('SENTRY_DNS'), + DSN: configService.getOrThrow('SENTRY_DSN'), INSTANCE_NAME: configService.getOrThrow('SENTRY_INSTANCE_NAME'), ENVIRONMENT: configService.getOrThrow('SENTRY_ENVIRONMENT'), }; @@ -46,7 +46,7 @@ function configureLoggingSentry( debug: true, serverName: sentryConfiguration.INSTANCE_NAME, environment: sentryConfiguration.ENVIRONMENT, - dsn: sentryConfiguration.DNS, + dsn: sentryConfiguration.DSN, integrations: [ // enable HTTP calls tracing new Sentry.Integrations.Console(), From df651436b8f7a24ac4348621fce34ad06bd4647e Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Thu, 29 Feb 2024 11:07:30 +0100 Subject: [PATCH 10/11] test: verify new schema in success case --- .../sqs/sqs-schema-generator.service.spec.ts | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/query/sqs/sqs-schema-generator.service.spec.ts b/src/query/sqs/sqs-schema-generator.service.spec.ts index 7310e9a..5160339 100644 --- a/src/query/sqs/sqs-schema-generator.service.spec.ts +++ b/src/query/sqs/sqs-schema-generator.service.spec.ts @@ -155,7 +155,52 @@ describe('SqsSchemaService', () => { service.updateSchema().subscribe({ next: () => { // then - expect(mockCouchDbClient.putDatabaseDocument).toHaveBeenCalled(); + expect(mockCouchDbClient.putDatabaseDocument).toHaveBeenCalledWith({ + documentId: '/_design/sqlite:config', + config: {}, + body: { + configVersion: + '2a26f7bc7e7e69940d811a4845a5f88374cbbb9868c8f4ce13303c3be71f2ad8', + language: 'sqlite', + sql: { + indexes: [], + options: { + table_name: { + field: '_id', + operation: 'prefix', + separator: ':', + }, + }, + tables: { + Child: { + fields: { + _id: 'TEXT', + _rev: 'TEXT', + age: 'INTEGER', + anonymized: 'INTEGER', + created: 'TEXT', + inactive: 'INTEGER', + name: 'TEXT', + updated: 'TEXT', + }, + }, + School: { + fields: { + _id: 'TEXT', + _rev: 'TEXT', + anonymized: 'INTEGER', + created: 'TEXT', + inactive: 'INTEGER', + name: 'TEXT', + numberOfStudents: 'INTEGER', + type: 'TEXT', + updated: 'TEXT', + }, + }, + }, + }, + }, + }); done(); }, error: (err) => { From e245babcb919ee56a5062b6f161cfaf146d8fb16 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Thu, 29 Feb 2024 11:26:55 +0100 Subject: [PATCH 11/11] chore: rename Entity to EntityType --- src/query/core/entity-config-resolver.ts | 15 +++++++++++---- src/query/domain/EntityConfig.ts | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/query/core/entity-config-resolver.ts b/src/query/core/entity-config-resolver.ts index 87702cf..a3ed824 100644 --- a/src/query/core/entity-config-resolver.ts +++ b/src/query/core/entity-config-resolver.ts @@ -1,6 +1,10 @@ import { IEntityConfigResolver } from './entity-config-resolver.interface'; import { map, Observable } from 'rxjs'; -import { Entity, EntityAttribute, EntityConfig } from '../domain/EntityConfig'; +import { + EntityAttribute, + EntityConfig, + EntityType, +} from '../domain/EntityConfig'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; export interface AppConfigFile { @@ -39,7 +43,7 @@ export class EntityConfigResolver implements IEntityConfigResolver { const keys = Object.keys(config.data).filter((key) => key.startsWith('entity:'), ); - const entities: Entity[] = []; + const entities: EntityType[] = []; for (let i = 0; i < keys.length; i++) { entities.push(this.parseEntityConfig(keys[i], config)); } @@ -48,7 +52,10 @@ export class EntityConfigResolver implements IEntityConfigResolver { ); } - private parseEntityConfig(entityKey: string, config: AppConfigFile): Entity { + private parseEntityConfig( + entityKey: string, + config: AppConfigFile, + ): EntityType { const data = config.data[entityKey]; let label: string; @@ -67,6 +74,6 @@ export class EntityConfigResolver implements IEntityConfigResolver { ), ); - return new Entity(label, attributes); + return new EntityType(label, attributes); } } diff --git a/src/query/domain/EntityConfig.ts b/src/query/domain/EntityConfig.ts index e4de90c..36316b6 100644 --- a/src/query/domain/EntityConfig.ts +++ b/src/query/domain/EntityConfig.ts @@ -5,7 +5,7 @@ export class EntityAttribute { ) {} } -export class Entity { +export class EntityType { constructor( public label: string, public attributes: EntityAttribute[], @@ -15,6 +15,6 @@ export class Entity { export class EntityConfig { constructor( public version: string, - public entities: Entity[], + public entities: EntityType[], ) {} }