From 405ab09f90d790f5d0f5be055d68f51bdb550888 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Tue, 13 Feb 2024 17:44:22 +0100 Subject: [PATCH 01/18] outlined code structure for report changes detection --- src/app.module.ts | 4 + src/couchdb/couch-db-client.service.ts | 23 +++++- src/couchdb/dtos.ts | 35 +++++++++ src/domain/report-data-change-event.ts | 12 +++ src/notification/notification.module.ts | 7 ++ .../notification/notification.service.spec.ts | 18 +++++ .../notification/notification.service.ts | 27 +++++++ .../report-change-detector.spec.ts | 55 ++++++++++++++ .../report-change.detector.ts | 45 ++++++++++++ src/report-changes/report-changes.module.ts | 10 +++ .../report-changes.service.spec.ts | 50 +++++++++++++ src/report-changes/report-changes.service.ts | 73 +++++++++++++++++++ .../repository/report-repository.service.ts | 9 +-- 13 files changed, 360 insertions(+), 8 deletions(-) create mode 100644 src/domain/report-data-change-event.ts create mode 100644 src/notification/notification.module.ts create mode 100644 src/notification/notification/notification.service.spec.ts create mode 100644 src/notification/notification/notification.service.ts create mode 100644 src/report-changes/report-change-detector/report-change-detector.spec.ts create mode 100644 src/report-changes/report-change-detector/report-change.detector.ts create mode 100644 src/report-changes/report-changes.module.ts create mode 100644 src/report-changes/report-changes.service.spec.ts create mode 100644 src/report-changes/report-changes.service.ts diff --git a/src/app.module.ts b/src/app.module.ts index 2f5b0d5..658cb5a 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -6,6 +6,8 @@ import { ConfigModule, ConfigService } from '@nestjs/config'; import { HttpModule } from '@nestjs/axios'; import { ReportModule } from './report/report.module'; import { ScheduleModule } from '@nestjs/schedule'; +import { ReportChangesModule } from './report-changes/report-changes.module'; +import { NotificationModule } from './notification/notification.module'; const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; @@ -59,6 +61,8 @@ const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; }, }), ReportModule, + ReportChangesModule, + NotificationModule, ], controllers: [], }) diff --git a/src/couchdb/couch-db-client.service.ts b/src/couchdb/couch-db-client.service.ts index ea3d252..7d1699b 100644 --- a/src/couchdb/couch-db-client.service.ts +++ b/src/couchdb/couch-db-client.service.ts @@ -2,12 +2,14 @@ import { Injectable, Logger } from '@nestjs/common'; import { catchError, map, Observable, of, switchMap } from 'rxjs'; import { HttpService } from '@nestjs/axios'; import { AxiosHeaders } from 'axios'; +import { CouchDbChangesResponse } from "./dtos"; @Injectable() export class CouchDbClient { private readonly logger = new Logger(CouchDbClient.name); - constructor(private httpService: HttpService) {} + constructor(private httpService: HttpService) { + } headDatabaseDocument( databaseUrl: string, @@ -118,4 +120,23 @@ export class CouchDbClient { private handleError(err: any) { this.logger.debug(err); } + + + changes( + databaseUrl: string, + databaseName: string, + config?: any, + ): Observable { + return this.httpService + .get(`${databaseUrl}/${databaseName}/_changes`, config) + .pipe( + map((response) => { + return response.data; + }), + catchError((err) => { + this.handleError(err); + throw err; + }), + ); + } } diff --git a/src/couchdb/dtos.ts b/src/couchdb/dtos.ts index dcacffe..03376f4 100644 --- a/src/couchdb/dtos.ts +++ b/src/couchdb/dtos.ts @@ -26,3 +26,38 @@ export class FindResponse { docs: T[]; } + +/** + * Response from the CouchDB changes endpoint, listing database docs that have changed + * since the given last change (last_seq). + * + * see https://docs.couchdb.org/en/stable/api/database/changes.html + */ +export interface CouchDbChangesResponse { + /** Last change update sequence */ + last_seq: string; + + /** array of docs with changes */ + results: CouchDbChangeResult[]; + + /** Count of remaining items in the feed */ + pending: number; +} + +/** + * A single result entry from a CouchDB changes feed, + * indicating one doc has changed. + * + * see https://docs.couchdb.org/en/stable/api/database/changes.html + */ +export interface CouchDbChangeResult { + /** _id of a doc with changes */ + id: string; + + /** List of document’s leaves with single field rev. */ + changes: { rev: string }[]; + + seq: string; + + doc?: any; +} \ No newline at end of file diff --git a/src/domain/report-data-change-event.ts b/src/domain/report-data-change-event.ts new file mode 100644 index 0000000..1574bf8 --- /dev/null +++ b/src/domain/report-data-change-event.ts @@ -0,0 +1,12 @@ +import { Reference } from "./reference"; + +/** + * Used as notification that a report's calculated results have changed, due to updates in the underlying database. + */ +export interface ReportDataChangeEvent { + /** The report for which data has changed */ + report: Reference; + + /** The calculation containing the latest data after the change, ready to be fetched */ + calculation: Reference; +} \ No newline at end of file diff --git a/src/notification/notification.module.ts b/src/notification/notification.module.ts new file mode 100644 index 0000000..d21a7fc --- /dev/null +++ b/src/notification/notification.module.ts @@ -0,0 +1,7 @@ +import { Module } from '@nestjs/common'; +import { NotificationService } from './notification/notification.service'; + +@Module({ + providers: [NotificationService] +}) +export class NotificationModule {} diff --git a/src/notification/notification/notification.service.spec.ts b/src/notification/notification/notification.service.spec.ts new file mode 100644 index 0000000..65bd59d --- /dev/null +++ b/src/notification/notification/notification.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { NotificationService } from './notification.service'; + +describe('NotificationService', () => { + let service: NotificationService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [NotificationService], + }).compile(); + + service = module.get(NotificationService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/notification/notification/notification.service.ts b/src/notification/notification/notification.service.ts new file mode 100644 index 0000000..d5f28aa --- /dev/null +++ b/src/notification/notification/notification.service.ts @@ -0,0 +1,27 @@ +import { Injectable } from '@nestjs/common'; +import { Observable, of } from "rxjs"; +import { Reference } from "../../domain/reference"; +import { ReportDataChangeEvent } from "../../domain/report-data-change-event"; + +/** + * Manage notification subscriptions and delivering events to subscribers. + */ +@Injectable() +export class NotificationService { + /** + * Get the list of reports for which notifications are subscribed by at least one client. + */ + activeReports(): Observable { + // TODO: is this emitting the whole list every time the subscriptions change, as the name suggests? + // or individual id when added (but then, how is unsubscribe tracked?) + // may be easier if I can just directly get the list of currently active reports + return of([]) + }; + + /** + * Trigger a notification event for the given report to any active subscribers. + */ + triggerNotification(event: ReportDataChangeEvent): void { + + } +} diff --git a/src/report-changes/report-change-detector/report-change-detector.spec.ts b/src/report-changes/report-change-detector/report-change-detector.spec.ts new file mode 100644 index 0000000..f9a6a68 --- /dev/null +++ b/src/report-changes/report-change-detector/report-change-detector.spec.ts @@ -0,0 +1,55 @@ +import { EntityDoc, ReportChangeDetector } from './report-change.detector'; +import { ReportDoc } from "../../report/repository/report-repository.service"; + +describe('ReportChangeDetectorService', () => { + + function testReportChangeDetection(sqlStatement: string, testCases: [EntityDoc, boolean][]) { + const report: Partial = { + _id: 'test-report-id', + mode: 'sql', + aggregationDefinitions: [sqlStatement], + }; + const service = new ReportChangeDetector(report as ReportDoc); + + for (const [docChange, expectedResult] of testCases) { + expect(service.affectsReport(docChange)).toBe(expectedResult); + } + } + + it('should detect doc change that triggers report change for basic "SELECT *" report', () => { + testReportChangeDetection("SELECT * FROM EventNote", [ + [{_id: "EventNote:1"}, true], + [{_id: "Event:1"}, false], + ]); + }); + + it('should detect only docs used in SELECT clause are relevant', () => { + // TODO: actually need to look at doc before and after change for the detection! + + testReportChangeDetection("SELECT name FROM EventNote", [ + [{_id: "EventNote:1", name: "foo"}, true], + [{_id: "EventNote:field-missing"}, false], + [{_id: "Event:other-type", name: "foo"}, false], + ]); + }); + + it('should detect only docs with previous or new value of field matching WHERE clause are relevant', () => { + // TODO: actually need to look at doc before and after change for the detection! + + testReportChangeDetection("SELECT * FROM EventNote WHERE location='Berlin'", [ + [{_id: "EventNote:1", location: "Berlin"}, true], + [{_id: "EventNote:field-not-matching", location: "New York"}, false], + [{_id: "EventNote:field-missing"}, false], + [{_id: "Event:other-type", location: "Berlin"}, false], + ]); + }); + + it('should detect fields in joins and complex json properties', () => { + testReportChangeDetection("SELECT c.name as Name, AttStatus as Status FROM Child c JOIN (SELECT json_extract(att.value, '$[0]') AS attendanceChildId, json_extract(att.value, '$[1].status') AS AttStatus FROM EventNote e, json_each(e.childrenAttendance) att) ON attendanceChildId=c._id", [ + // TODO + [{_id: "EventNote:1"}, true], + [{_id: "Child:1"}, true], + ]); + }); + +}); \ No newline at end of file diff --git a/src/report-changes/report-change-detector/report-change.detector.ts b/src/report-changes/report-change-detector/report-change.detector.ts new file mode 100644 index 0000000..7b700b7 --- /dev/null +++ b/src/report-changes/report-change-detector/report-change.detector.ts @@ -0,0 +1,45 @@ +import { ReportDoc } from "../../report/repository/report-repository.service"; + +export class ReportChangeDetector { + private report?: ReportDoc; + private sqlTableNames: string[] = []; + + constructor(report: ReportDoc) { + this.updateReportConfig(report); + } + + updateReportConfig(report: ReportDoc) { + this.report = report; + + this.sqlTableNames = this.getSqlTableNames(report); + } + + private getSqlTableNames(report: ReportDoc) { + const sqlFromTableRegex = /FROM\s+(\w+)/g; + + return report.aggregationDefinitions.map((sql: string) => + [...sql.matchAll(sqlFromTableRegex)].map(match => match[1] /* matching regex group (table name) */) + ).flat(); + } + + affectsReport(doc: EntityDoc): boolean { + const entityType = doc._id.split(":")[0]; + if (this.sqlTableNames.includes(entityType)) { + + // TODO: better detection if doc affects report + + return true; + } + + return false; + } +} + +/** + * A doc in the database representing an entity managed in the frontend. + */ +export interface EntityDoc { + _id: string; + + [key: string]: any; +} \ No newline at end of file diff --git a/src/report-changes/report-changes.module.ts b/src/report-changes/report-changes.module.ts new file mode 100644 index 0000000..e21ece6 --- /dev/null +++ b/src/report-changes/report-changes.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { ReportChangesService } from "./report-changes.service"; + +@Module({ + providers: [ + ReportChangesService + ], +}) +export class ReportChangesModule { +} diff --git a/src/report-changes/report-changes.service.spec.ts b/src/report-changes/report-changes.service.spec.ts new file mode 100644 index 0000000..aa625b6 --- /dev/null +++ b/src/report-changes/report-changes.service.spec.ts @@ -0,0 +1,50 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReportChangesService } from './report-changes.service'; +import { BehaviorSubject, map } from "rxjs"; +import { NotificationService } from "../notification/notification/notification.service"; +import { Reference } from "../domain/reference"; + +describe('ReportChangesService', () => { + let service: ReportChangesService; + let mockNotificationService: Partial; + + let activeReports: BehaviorSubject; + + beforeEach(async () => { + activeReports = new BehaviorSubject([]); + mockNotificationService = { + activeReports: () => activeReports.asObservable() + .pipe(map(reportIds => reportIds.map(id => new Reference(id)))), + triggerNotification: jest.fn() + } + + const module: TestingModule = await Test.createTestingModule({ + providers: [ReportChangesService, {provide: NotificationService, useValue: mockNotificationService}], + }).compile(); + + service = module.get(ReportChangesService); + }); + + it('should trigger notification after adding active report through NotificationService', (done) => { + const testReportId = "report1"; + activeReports.next([testReportId]); + + // TODO mock a couchDbService.changes event + + (mockNotificationService.triggerNotification as jest.Mock).mockImplementation((reportId: string) => { + expect(reportId).toBe(testReportId); + done(); + }); + }); + + it('should trigger notification after adding active report through NotificationService', async () => { + activeReports.next(["report1"]); + activeReports.next(["report2" /* removed report1 */]); + + // TODO mock a couchDbService.changes event + + await new Promise(process.nextTick); // wait for any async operations to finish + expect(mockNotificationService.triggerNotification).not.toHaveBeenCalledWith("report1"); + expect(mockNotificationService.triggerNotification).toHaveBeenCalledWith("report2"); + }); +}); diff --git a/src/report-changes/report-changes.service.ts b/src/report-changes/report-changes.service.ts new file mode 100644 index 0000000..40f6bdf --- /dev/null +++ b/src/report-changes/report-changes.service.ts @@ -0,0 +1,73 @@ +import { Injectable } from '@nestjs/common'; +import { ReportChangeDetector } from "./report-change-detector/report-change.detector"; +import { ReportDoc, ReportRepository } from "../report/repository/report-repository.service"; +import { NotificationService } from "../notification/notification/notification.service"; +import { Reference } from "../domain/reference"; +import { ReportDataChangeEvent } from "../domain/report-data-change-event"; +import { ReportCalculation } from "../domain/report-calculation"; +import { CouchDbClient } from "../couchdb/couch-db-client.service"; +import { CouchDbChangeResult, CouchDbChangesResponse } from "../couchdb/dtos"; + +/** + * Monitor all changes to the application database and check if they affect any report's results. + */ +@Injectable() +export class ReportChangesService { + private reportMonitors = new Map(); + + constructor( + private notificationService: NotificationService, + private reportRepository: ReportRepository, + private couchDbClient: CouchDbClient + ) { + // TODO: where to get databaseUrl and databaseName from? Can we centralize this ...? + this.couchDbClient.changes("TODO", "app",).subscribe((changes: CouchDbChangesResponse) => { + // TODO: ensure continued fetching until all changes done + // TODO: collect a batch of changes for a while before checking? + for (const c of changes.results) { + this.checkIncomingDocChange(c) + + if (this.reportMonitors.has(c.id)) { + // TODO: reportId in reportMonitors with or without prefix? + + // TODO: load actual current doc (may not be in c.doc?) + this.reportMonitors.get(c.id)?.updateReportConfig(c.doc); + } + } + }); + + this.notificationService.activeReports().subscribe((reports: Reference[]) => reports.forEach(r => this.registerReportMonitoring(r))); + } + + async registerReportMonitoring(report: Reference) { + if (!this.reportMonitors.has(report.id)) { + // TODO: reuse ReportRepository and its ReportDoc (currently not exported) here? Or duplicate some things to keep stuff isolated? + // TODO: can we centralize the auth stuff somehow? not sure where I would get this from in this context ... + this.reportRepository.fetchReport("TODO", report.id) + .subscribe((reportDoc: ReportDoc) => this.reportMonitors.set(report.id, new ReportChangeDetector(reportDoc))); + } + } + + private checkIncomingDocChange(change: CouchDbChangeResult) { + const doc = {_id: ""}// TODO: load doc here? + for (const [reportId, changeDetector] of this.reportMonitors.entries()) { + if (!changeDetector.affectsReport(doc)) { + continue; + } + + const reportRef = new Reference(reportId); + + // TODO: calculate a new report calculation here? Or in the changeDetector? + // const newResult = await this.reportService.runReportCalculation(reportId); + // if (newResult.hash !== oldResult.hash) + const calculation: ReportCalculation = new ReportCalculation("x", reportRef); + + const event: ReportDataChangeEvent = { + report: reportRef, + calculation: reportRef + }; + this.notificationService.triggerNotification(event); + } + } +} + diff --git a/src/report/repository/report-repository.service.ts b/src/report/repository/report-repository.service.ts index d4955cc..73a9186 100644 --- a/src/report/repository/report-repository.service.ts +++ b/src/report/repository/report-repository.service.ts @@ -1,15 +1,10 @@ -import { - ForbiddenException, - Injectable, - NotFoundException, - UnauthorizedException, -} from '@nestjs/common'; +import { ForbiddenException, Injectable, NotFoundException, UnauthorizedException, } from '@nestjs/common'; import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; import { catchError, map, Observable } from 'rxjs'; import { CouchDbRow } from '../../couchdb/dtos'; -interface ReportDoc { +export interface ReportDoc { _id: string; _rev: string; title: string; From 53e9035a3025fa033286284a20c39ab9bcc1c005 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Wed, 14 Feb 2024 11:54:04 +0100 Subject: [PATCH 02/18] fix: use Report object instead of exposing ReportDoc --- src/domain/report.ts | 9 ++- .../report-change-detector.spec.ts | 57 ++++++++------- .../report-change.detector.ts | 25 ++++--- src/report-changes/report-changes.service.ts | 71 ++++++++++++------- .../repository/report-repository.service.ts | 15 ++-- src/report/storage/report-storage.service.ts | 12 +++- 6 files changed, 118 insertions(+), 71 deletions(-) diff --git a/src/domain/report.ts b/src/domain/report.ts index 1d4102d..2d1c327 100644 --- a/src/domain/report.ts +++ b/src/domain/report.ts @@ -13,10 +13,12 @@ export class Report { id: string; name: string; schema: ReportSchema | undefined; + queries: string[]; - constructor(id: string, name: string) { + constructor(id: string, name: string, queries: string[]) { this.id = id; this.name = name; + this.queries = queries; } setId(id: string): Report { @@ -33,4 +35,9 @@ export class Report { this.schema = schema; return this; } + + setQueries(queries: string[]): Report { + this.queries = queries; + return this; + } } diff --git a/src/report-changes/report-change-detector/report-change-detector.spec.ts b/src/report-changes/report-change-detector/report-change-detector.spec.ts index f9a6a68..a824911 100644 --- a/src/report-changes/report-change-detector/report-change-detector.spec.ts +++ b/src/report-changes/report-change-detector/report-change-detector.spec.ts @@ -1,15 +1,17 @@ import { EntityDoc, ReportChangeDetector } from './report-change.detector'; -import { ReportDoc } from "../../report/repository/report-repository.service"; +import { Report } from '../../report/repository/report-repository.service'; describe('ReportChangeDetectorService', () => { - - function testReportChangeDetection(sqlStatement: string, testCases: [EntityDoc, boolean][]) { - const report: Partial = { + function testReportChangeDetection( + sqlStatement: string, + testCases: [EntityDoc, boolean][], + ) { + const report: Partial = { _id: 'test-report-id', mode: 'sql', aggregationDefinitions: [sqlStatement], }; - const service = new ReportChangeDetector(report as ReportDoc); + const service = new ReportChangeDetector(report as Report); for (const [docChange, expectedResult] of testCases) { expect(service.affectsReport(docChange)).toBe(expectedResult); @@ -17,39 +19,44 @@ describe('ReportChangeDetectorService', () => { } it('should detect doc change that triggers report change for basic "SELECT *" report', () => { - testReportChangeDetection("SELECT * FROM EventNote", [ - [{_id: "EventNote:1"}, true], - [{_id: "Event:1"}, false], + testReportChangeDetection('SELECT * FROM EventNote', [ + [{ _id: 'EventNote:1' }, true], + [{ _id: 'Event:1' }, false], ]); }); it('should detect only docs used in SELECT clause are relevant', () => { // TODO: actually need to look at doc before and after change for the detection! - testReportChangeDetection("SELECT name FROM EventNote", [ - [{_id: "EventNote:1", name: "foo"}, true], - [{_id: "EventNote:field-missing"}, false], - [{_id: "Event:other-type", name: "foo"}, false], + testReportChangeDetection('SELECT name FROM EventNote', [ + [{ _id: 'EventNote:1', name: 'foo' }, true], + [{ _id: 'EventNote:field-missing' }, false], + [{ _id: 'Event:other-type', name: 'foo' }, false], ]); }); it('should detect only docs with previous or new value of field matching WHERE clause are relevant', () => { // TODO: actually need to look at doc before and after change for the detection! - testReportChangeDetection("SELECT * FROM EventNote WHERE location='Berlin'", [ - [{_id: "EventNote:1", location: "Berlin"}, true], - [{_id: "EventNote:field-not-matching", location: "New York"}, false], - [{_id: "EventNote:field-missing"}, false], - [{_id: "Event:other-type", location: "Berlin"}, false], - ]); + testReportChangeDetection( + "SELECT * FROM EventNote WHERE location='Berlin'", + [ + [{ _id: 'EventNote:1', location: 'Berlin' }, true], + [{ _id: 'EventNote:field-not-matching', location: 'New York' }, false], + [{ _id: 'EventNote:field-missing' }, false], + [{ _id: 'Event:other-type', location: 'Berlin' }, false], + ], + ); }); it('should detect fields in joins and complex json properties', () => { - testReportChangeDetection("SELECT c.name as Name, AttStatus as Status FROM Child c JOIN (SELECT json_extract(att.value, '$[0]') AS attendanceChildId, json_extract(att.value, '$[1].status') AS AttStatus FROM EventNote e, json_each(e.childrenAttendance) att) ON attendanceChildId=c._id", [ - // TODO - [{_id: "EventNote:1"}, true], - [{_id: "Child:1"}, true], - ]); + testReportChangeDetection( + "SELECT c.name as Name, AttStatus as Status FROM Child c JOIN (SELECT json_extract(att.value, '$[0]') AS attendanceChildId, json_extract(att.value, '$[1].status') AS AttStatus FROM EventNote e, json_each(e.childrenAttendance) att) ON attendanceChildId=c._id", + [ + // TODO + [{ _id: 'EventNote:1' }, true], + [{ _id: 'Child:1' }, true], + ], + ); }); - -}); \ No newline at end of file +}); diff --git a/src/report-changes/report-change-detector/report-change.detector.ts b/src/report-changes/report-change-detector/report-change.detector.ts index 7b700b7..40135c0 100644 --- a/src/report-changes/report-change-detector/report-change.detector.ts +++ b/src/report-changes/report-change-detector/report-change.detector.ts @@ -1,31 +1,34 @@ -import { ReportDoc } from "../../report/repository/report-repository.service"; +import { Report } from '../../domain/report'; export class ReportChangeDetector { - private report?: ReportDoc; + private report?: Report; private sqlTableNames: string[] = []; - constructor(report: ReportDoc) { + constructor(report: Report) { this.updateReportConfig(report); } - updateReportConfig(report: ReportDoc) { + updateReportConfig(report: Report) { this.report = report; this.sqlTableNames = this.getSqlTableNames(report); } - private getSqlTableNames(report: ReportDoc) { + private getSqlTableNames(report: Report) { const sqlFromTableRegex = /FROM\s+(\w+)/g; - return report.aggregationDefinitions.map((sql: string) => - [...sql.matchAll(sqlFromTableRegex)].map(match => match[1] /* matching regex group (table name) */) - ).flat(); + return report.queries + .map((sql: string) => + [...sql.matchAll(sqlFromTableRegex)].map( + (match) => match[1] /* matching regex group (table name) */, + ), + ) + .flat(); } affectsReport(doc: EntityDoc): boolean { - const entityType = doc._id.split(":")[0]; + const entityType = doc._id.split(':')[0]; if (this.sqlTableNames.includes(entityType)) { - // TODO: better detection if doc affects report return true; @@ -42,4 +45,4 @@ export interface EntityDoc { _id: string; [key: string]: any; -} \ No newline at end of file +} diff --git a/src/report-changes/report-changes.service.ts b/src/report-changes/report-changes.service.ts index 40f6bdf..96f10da 100644 --- a/src/report-changes/report-changes.service.ts +++ b/src/report-changes/report-changes.service.ts @@ -1,12 +1,15 @@ import { Injectable } from '@nestjs/common'; -import { ReportChangeDetector } from "./report-change-detector/report-change.detector"; -import { ReportDoc, ReportRepository } from "../report/repository/report-repository.service"; -import { NotificationService } from "../notification/notification/notification.service"; -import { Reference } from "../domain/reference"; -import { ReportDataChangeEvent } from "../domain/report-data-change-event"; -import { ReportCalculation } from "../domain/report-calculation"; -import { CouchDbClient } from "../couchdb/couch-db-client.service"; -import { CouchDbChangeResult, CouchDbChangesResponse } from "../couchdb/dtos"; +import { ReportChangeDetector } from './report-change-detector/report-change.detector'; +import { + Report, + ReportRepository, +} from '../report/repository/report-repository.service'; +import { NotificationService } from '../notification/notification/notification.service'; +import { Reference } from '../domain/reference'; +import { ReportDataChangeEvent } from '../domain/report-data-change-event'; +import { ReportCalculation } from '../domain/report-calculation'; +import { CouchDbClient } from '../couchdb/couch-db-client.service'; +import { CouchDbChangeResult, CouchDbChangesResponse } from '../couchdb/dtos'; /** * Monitor all changes to the application database and check if they affect any report's results. @@ -18,38 +21,50 @@ export class ReportChangesService { constructor( private notificationService: NotificationService, private reportRepository: ReportRepository, - private couchDbClient: CouchDbClient + private couchDbClient: CouchDbClient, ) { // TODO: where to get databaseUrl and databaseName from? Can we centralize this ...? - this.couchDbClient.changes("TODO", "app",).subscribe((changes: CouchDbChangesResponse) => { - // TODO: ensure continued fetching until all changes done - // TODO: collect a batch of changes for a while before checking? - for (const c of changes.results) { - this.checkIncomingDocChange(c) + this.couchDbClient + .changes('TODO', 'app') + .subscribe((changes: CouchDbChangesResponse) => { + // TODO: ensure continued fetching until all changes done + // TODO: collect a batch of changes for a while before checking? + for (const c of changes.results) { + this.checkIncomingDocChange(c); - if (this.reportMonitors.has(c.id)) { - // TODO: reportId in reportMonitors with or without prefix? + if (this.reportMonitors.has(c.id)) { + // TODO: reportId in reportMonitors with or without prefix? - // TODO: load actual current doc (may not be in c.doc?) - this.reportMonitors.get(c.id)?.updateReportConfig(c.doc); + // TODO: load actual current doc (may not be in c.doc?) + this.reportMonitors.get(c.id)?.updateReportConfig(c.doc); + } } - } - }); + }); - this.notificationService.activeReports().subscribe((reports: Reference[]) => reports.forEach(r => this.registerReportMonitoring(r))); + this.notificationService + .activeReports() + .subscribe((reports: Reference[]) => + reports.forEach((r) => this.registerReportMonitoring(r)), + ); } async registerReportMonitoring(report: Reference) { if (!this.reportMonitors.has(report.id)) { // TODO: reuse ReportRepository and its ReportDoc (currently not exported) here? Or duplicate some things to keep stuff isolated? // TODO: can we centralize the auth stuff somehow? not sure where I would get this from in this context ... - this.reportRepository.fetchReport("TODO", report.id) - .subscribe((reportDoc: ReportDoc) => this.reportMonitors.set(report.id, new ReportChangeDetector(reportDoc))); + this.reportRepository + .fetchReport('TODO', report.id) + .subscribe((reportDoc: Report) => + this.reportMonitors.set( + report.id, + new ReportChangeDetector(reportDoc), + ), + ); } } private checkIncomingDocChange(change: CouchDbChangeResult) { - const doc = {_id: ""}// TODO: load doc here? + const doc = { _id: '' }; // TODO: load doc here? for (const [reportId, changeDetector] of this.reportMonitors.entries()) { if (!changeDetector.affectsReport(doc)) { continue; @@ -60,14 +75,16 @@ export class ReportChangesService { // TODO: calculate a new report calculation here? Or in the changeDetector? // const newResult = await this.reportService.runReportCalculation(reportId); // if (newResult.hash !== oldResult.hash) - const calculation: ReportCalculation = new ReportCalculation("x", reportRef); + const calculation: ReportCalculation = new ReportCalculation( + 'x', + reportRef, + ); const event: ReportDataChangeEvent = { report: reportRef, - calculation: reportRef + calculation: reportRef, }; this.notificationService.triggerNotification(event); } } } - diff --git a/src/report/repository/report-repository.service.ts b/src/report/repository/report-repository.service.ts index 73a9186..2bd6c45 100644 --- a/src/report/repository/report-repository.service.ts +++ b/src/report/repository/report-repository.service.ts @@ -1,10 +1,15 @@ -import { ForbiddenException, Injectable, NotFoundException, UnauthorizedException, } from '@nestjs/common'; +import { + ForbiddenException, + Injectable, + NotFoundException, + UnauthorizedException, +} from '@nestjs/common'; import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; import { catchError, map, Observable } from 'rxjs'; import { CouchDbRow } from '../../couchdb/dtos'; -export interface ReportDoc { +interface Report { _id: string; _rev: string; title: string; @@ -23,7 +28,7 @@ export interface ReportDoc { interface FetchReportsResponse { total_rows: number; offset: number; - rows: CouchDbRow[]; + rows: CouchDbRow[]; } @Injectable() @@ -63,9 +68,9 @@ export class ReportRepository { ); } - fetchReport(authToken: string, reportId: string): Observable { + fetchReport(authToken: string, reportId: string): Observable { return this.http - .get(`${this.dbUrl}/app/${reportId}`, { + .get(`${this.dbUrl}/app/${reportId}`, { headers: { Authorization: authToken, }, diff --git a/src/report/storage/report-storage.service.ts b/src/report/storage/report-storage.service.ts index d885bbe..564fcf7 100644 --- a/src/report/storage/report-storage.service.ts +++ b/src/report/storage/report-storage.service.ts @@ -31,7 +31,11 @@ export class DefaultReportStorage implements ReportStorage { return response.rows .filter((row) => row.doc.mode === mode) .map((reportEntity) => - new Report(reportEntity.id, reportEntity.doc.title).setSchema({ + new Report( + reportEntity.id, + reportEntity.doc.title, + reportEntity.doc.aggregationDefinitions, + ).setSchema({ fields: reportEntity.doc.aggregationDefinitions, // todo generate actual fields here }), ); @@ -42,7 +46,11 @@ export class DefaultReportStorage implements ReportStorage { fetchReport(authToken: string, reportRef: Reference): Observable { return this.reportRepository.fetchReport(authToken, reportRef.id).pipe( map((reportDoc) => { - return new Report(reportDoc._id, reportDoc.title).setSchema({ + return new Report( + reportDoc._id, + reportDoc.title, + reportDoc.aggregationDefinitions, + ).setSchema({ fields: reportDoc.aggregationDefinitions, // todo generate actual fields here }); }), From d7811aaaaae541d8d0de05755370bfe632f46efd Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Wed, 14 Feb 2024 12:08:49 +0100 Subject: [PATCH 03/18] chore: refactor folder structure --- src/domain/report-data-change-event.ts | 6 +- src/domain/report.ts | 6 ++ .../notification.service.spec.ts | 0 .../notification.service.ts | 18 +++--- src/notification/notification.module.ts | 4 +- .../couchdb-report-changes.service.spec.ts | 63 +++++++++++++++++++ .../couchdb-report-changes.service.ts} | 46 +++++++------- .../report-change-detector.spec.ts | 6 +- .../report-change.detector.ts | 0 .../core/report-changes.service.ts | 5 ++ src/report-changes/report-changes.module.ts | 9 +-- .../report-changes.service.spec.ts | 50 --------------- .../repository/report-repository.service.ts | 8 +-- src/report/storage/report-storage.service.ts | 10 +-- 14 files changed, 125 insertions(+), 106 deletions(-) rename src/notification/{notification => core}/notification.service.spec.ts (100%) rename src/notification/{notification => core}/notification.service.ts (57%) create mode 100644 src/report-changes/core/couchdb-report-changes.service.spec.ts rename src/report-changes/{report-changes.service.ts => core/couchdb-report-changes.service.ts} (68%) rename src/report-changes/{report-change-detector => core}/report-change-detector.spec.ts (93%) rename src/report-changes/{report-change-detector => core}/report-change.detector.ts (100%) create mode 100644 src/report-changes/core/report-changes.service.ts delete mode 100644 src/report-changes/report-changes.service.spec.ts diff --git a/src/domain/report-data-change-event.ts b/src/domain/report-data-change-event.ts index 1574bf8..2701c88 100644 --- a/src/domain/report-data-change-event.ts +++ b/src/domain/report-data-change-event.ts @@ -1,7 +1,7 @@ -import { Reference } from "./reference"; +import { Reference } from './reference'; /** - * Used as notification that a report's calculated results have changed, due to updates in the underlying database. + * Used as core that a report's calculated results have changed, due to updates in the underlying database. */ export interface ReportDataChangeEvent { /** The report for which data has changed */ @@ -9,4 +9,4 @@ export interface ReportDataChangeEvent { /** The calculation containing the latest data after the change, ready to be fetched */ calculation: Reference; -} \ No newline at end of file +} diff --git a/src/domain/report.ts b/src/domain/report.ts index 2d1c327..d800e62 100644 --- a/src/domain/report.ts +++ b/src/domain/report.ts @@ -12,6 +12,7 @@ export interface ReportSchema { export class Report { id: string; name: string; + mode: string | undefined; schema: ReportSchema | undefined; queries: string[]; @@ -26,6 +27,11 @@ export class Report { return this; } + setMode(mode: string): Report { + this.mode = mode; + return this; + } + setName(name: string): Report { this.name = name; return this; diff --git a/src/notification/notification/notification.service.spec.ts b/src/notification/core/notification.service.spec.ts similarity index 100% rename from src/notification/notification/notification.service.spec.ts rename to src/notification/core/notification.service.spec.ts diff --git a/src/notification/notification/notification.service.ts b/src/notification/core/notification.service.ts similarity index 57% rename from src/notification/notification/notification.service.ts rename to src/notification/core/notification.service.ts index d5f28aa..d5b2a17 100644 --- a/src/notification/notification/notification.service.ts +++ b/src/notification/core/notification.service.ts @@ -1,10 +1,10 @@ import { Injectable } from '@nestjs/common'; -import { Observable, of } from "rxjs"; -import { Reference } from "../../domain/reference"; -import { ReportDataChangeEvent } from "../../domain/report-data-change-event"; +import { Observable, of } from 'rxjs'; +import { Reference } from '../../domain/reference'; +import { ReportDataChangeEvent } from '../../domain/report-data-change-event'; /** - * Manage notification subscriptions and delivering events to subscribers. + * Manage core subscriptions and delivering events to subscribers. */ @Injectable() export class NotificationService { @@ -15,13 +15,11 @@ export class NotificationService { // TODO: is this emitting the whole list every time the subscriptions change, as the name suggests? // or individual id when added (but then, how is unsubscribe tracked?) // may be easier if I can just directly get the list of currently active reports - return of([]) - }; + return of([]); + } /** - * Trigger a notification event for the given report to any active subscribers. + * Trigger a core event for the given report to any active subscribers. */ - triggerNotification(event: ReportDataChangeEvent): void { - - } + triggerNotification(event: ReportDataChangeEvent): void {} } diff --git a/src/notification/notification.module.ts b/src/notification/notification.module.ts index d21a7fc..815726c 100644 --- a/src/notification/notification.module.ts +++ b/src/notification/notification.module.ts @@ -1,7 +1,7 @@ import { Module } from '@nestjs/common'; -import { NotificationService } from './notification/notification.service'; +import { NotificationService } from './core/notification.service'; @Module({ - providers: [NotificationService] + providers: [NotificationService], }) export class NotificationModule {} diff --git a/src/report-changes/core/couchdb-report-changes.service.spec.ts b/src/report-changes/core/couchdb-report-changes.service.spec.ts new file mode 100644 index 0000000..1901157 --- /dev/null +++ b/src/report-changes/core/couchdb-report-changes.service.spec.ts @@ -0,0 +1,63 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { CouchdbReportChangesService } from './couchdb-report-changes.service'; +import { BehaviorSubject, map } from 'rxjs'; +import { NotificationService } from '../../notification/core/notification.service'; +import { Reference } from '../../domain/reference'; + +describe('CouchdbReportChangesService', () => { + let service: CouchdbReportChangesService; + let mockNotificationService: Partial; + + let activeReports: BehaviorSubject; + + beforeEach(async () => { + activeReports = new BehaviorSubject([]); + mockNotificationService = { + activeReports: () => + activeReports + .asObservable() + .pipe(map((reportIds) => reportIds.map((id) => new Reference(id)))), + triggerNotification: jest.fn(), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + CouchdbReportChangesService, + { provide: NotificationService, useValue: mockNotificationService }, + ], + }).compile(); + + service = module.get( + CouchdbReportChangesService, + ); + }); + + it('should trigger core after adding active report through NotificationService', (done) => { + const testReportId = 'report1'; + activeReports.next([testReportId]); + + // TODO mock a couchDbService.changes event + + ( + mockNotificationService.triggerNotification as jest.Mock + ).mockImplementation((reportId: string) => { + expect(reportId).toBe(testReportId); + done(); + }); + }); + + it('should trigger core after adding active report through NotificationService', async () => { + activeReports.next(['report1']); + activeReports.next(['report2' /* removed report1 */]); + + // TODO mock a couchDbService.changes event + + await new Promise(process.nextTick); // wait for any async operations to finish + expect( + mockNotificationService.triggerNotification, + ).not.toHaveBeenCalledWith('report1'); + expect(mockNotificationService.triggerNotification).toHaveBeenCalledWith( + 'report2', + ); + }); +}); diff --git a/src/report-changes/report-changes.service.ts b/src/report-changes/core/couchdb-report-changes.service.ts similarity index 68% rename from src/report-changes/report-changes.service.ts rename to src/report-changes/core/couchdb-report-changes.service.ts index 96f10da..284388a 100644 --- a/src/report-changes/report-changes.service.ts +++ b/src/report-changes/core/couchdb-report-changes.service.ts @@ -1,26 +1,25 @@ import { Injectable } from '@nestjs/common'; -import { ReportChangeDetector } from './report-change-detector/report-change.detector'; +import { ReportChangeDetector } from './report-change.detector'; +import { NotificationService } from '../../notification/core/notification.service'; +import { Reference } from '../../domain/reference'; +import { ReportDataChangeEvent } from '../../domain/report-data-change-event'; +import { ReportCalculation } from '../../domain/report-calculation'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; import { - Report, - ReportRepository, -} from '../report/repository/report-repository.service'; -import { NotificationService } from '../notification/notification/notification.service'; -import { Reference } from '../domain/reference'; -import { ReportDataChangeEvent } from '../domain/report-data-change-event'; -import { ReportCalculation } from '../domain/report-calculation'; -import { CouchDbClient } from '../couchdb/couch-db-client.service'; -import { CouchDbChangeResult, CouchDbChangesResponse } from '../couchdb/dtos'; + CouchDbChangeResult, + CouchDbChangesResponse, +} from '../../couchdb/dtos'; +import { Report } from '../../domain/report'; +import { ReportChangesService } from './report-changes.service'; +import { ReportStorage } from '../../report/core/report-storage'; -/** - * Monitor all changes to the application database and check if they affect any report's results. - */ @Injectable() -export class ReportChangesService { +export class CouchdbReportChangesService implements ReportChangesService { private reportMonitors = new Map(); constructor( private notificationService: NotificationService, - private reportRepository: ReportRepository, + private reportStorage: ReportStorage, private couchDbClient: CouchDbClient, ) { // TODO: where to get databaseUrl and databaseName from? Can we centralize this ...? @@ -52,14 +51,15 @@ export class ReportChangesService { if (!this.reportMonitors.has(report.id)) { // TODO: reuse ReportRepository and its ReportDoc (currently not exported) here? Or duplicate some things to keep stuff isolated? // TODO: can we centralize the auth stuff somehow? not sure where I would get this from in this context ... - this.reportRepository - .fetchReport('TODO', report.id) - .subscribe((reportDoc: Report) => - this.reportMonitors.set( - report.id, - new ReportChangeDetector(reportDoc), - ), - ); + this.reportStorage + .fetchReport('TODO', report) + .subscribe((report: Report | undefined) => { + if (!report) { + return; + } + + this.reportMonitors.set(report.id, new ReportChangeDetector(report)); + }); } } diff --git a/src/report-changes/report-change-detector/report-change-detector.spec.ts b/src/report-changes/core/report-change-detector.spec.ts similarity index 93% rename from src/report-changes/report-change-detector/report-change-detector.spec.ts rename to src/report-changes/core/report-change-detector.spec.ts index a824911..341932b 100644 --- a/src/report-changes/report-change-detector/report-change-detector.spec.ts +++ b/src/report-changes/core/report-change-detector.spec.ts @@ -1,5 +1,5 @@ import { EntityDoc, ReportChangeDetector } from './report-change.detector'; -import { Report } from '../../report/repository/report-repository.service'; +import { Report } from '../../domain/report'; describe('ReportChangeDetectorService', () => { function testReportChangeDetection( @@ -7,9 +7,9 @@ describe('ReportChangeDetectorService', () => { testCases: [EntityDoc, boolean][], ) { const report: Partial = { - _id: 'test-report-id', + id: 'test-report-id', mode: 'sql', - aggregationDefinitions: [sqlStatement], + queries: [sqlStatement], }; const service = new ReportChangeDetector(report as Report); diff --git a/src/report-changes/report-change-detector/report-change.detector.ts b/src/report-changes/core/report-change.detector.ts similarity index 100% rename from src/report-changes/report-change-detector/report-change.detector.ts rename to src/report-changes/core/report-change.detector.ts diff --git a/src/report-changes/core/report-changes.service.ts b/src/report-changes/core/report-changes.service.ts new file mode 100644 index 0000000..74916b2 --- /dev/null +++ b/src/report-changes/core/report-changes.service.ts @@ -0,0 +1,5 @@ +/** + * Monitor all changes to the application database and check if they affect any report's results. + */ + +export interface ReportChangesService {} diff --git a/src/report-changes/report-changes.module.ts b/src/report-changes/report-changes.module.ts index e21ece6..c45cc44 100644 --- a/src/report-changes/report-changes.module.ts +++ b/src/report-changes/report-changes.module.ts @@ -1,10 +1,7 @@ import { Module } from '@nestjs/common'; -import { ReportChangesService } from "./report-changes.service"; +import { CouchdbReportChangesService } from './core/couchdb-report-changes.service'; @Module({ - providers: [ - ReportChangesService - ], + providers: [CouchdbReportChangesService], }) -export class ReportChangesModule { -} +export class ReportChangesModule {} diff --git a/src/report-changes/report-changes.service.spec.ts b/src/report-changes/report-changes.service.spec.ts deleted file mode 100644 index aa625b6..0000000 --- a/src/report-changes/report-changes.service.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { ReportChangesService } from './report-changes.service'; -import { BehaviorSubject, map } from "rxjs"; -import { NotificationService } from "../notification/notification/notification.service"; -import { Reference } from "../domain/reference"; - -describe('ReportChangesService', () => { - let service: ReportChangesService; - let mockNotificationService: Partial; - - let activeReports: BehaviorSubject; - - beforeEach(async () => { - activeReports = new BehaviorSubject([]); - mockNotificationService = { - activeReports: () => activeReports.asObservable() - .pipe(map(reportIds => reportIds.map(id => new Reference(id)))), - triggerNotification: jest.fn() - } - - const module: TestingModule = await Test.createTestingModule({ - providers: [ReportChangesService, {provide: NotificationService, useValue: mockNotificationService}], - }).compile(); - - service = module.get(ReportChangesService); - }); - - it('should trigger notification after adding active report through NotificationService', (done) => { - const testReportId = "report1"; - activeReports.next([testReportId]); - - // TODO mock a couchDbService.changes event - - (mockNotificationService.triggerNotification as jest.Mock).mockImplementation((reportId: string) => { - expect(reportId).toBe(testReportId); - done(); - }); - }); - - it('should trigger notification after adding active report through NotificationService', async () => { - activeReports.next(["report1"]); - activeReports.next(["report2" /* removed report1 */]); - - // TODO mock a couchDbService.changes event - - await new Promise(process.nextTick); // wait for any async operations to finish - expect(mockNotificationService.triggerNotification).not.toHaveBeenCalledWith("report1"); - expect(mockNotificationService.triggerNotification).toHaveBeenCalledWith("report2"); - }); -}); diff --git a/src/report/repository/report-repository.service.ts b/src/report/repository/report-repository.service.ts index 2bd6c45..d4955cc 100644 --- a/src/report/repository/report-repository.service.ts +++ b/src/report/repository/report-repository.service.ts @@ -9,7 +9,7 @@ import { ConfigService } from '@nestjs/config'; import { catchError, map, Observable } from 'rxjs'; import { CouchDbRow } from '../../couchdb/dtos'; -interface Report { +interface ReportDoc { _id: string; _rev: string; title: string; @@ -28,7 +28,7 @@ interface Report { interface FetchReportsResponse { total_rows: number; offset: number; - rows: CouchDbRow[]; + rows: CouchDbRow[]; } @Injectable() @@ -68,9 +68,9 @@ export class ReportRepository { ); } - fetchReport(authToken: string, reportId: string): Observable { + fetchReport(authToken: string, reportId: string): Observable { return this.http - .get(`${this.dbUrl}/app/${reportId}`, { + .get(`${this.dbUrl}/app/${reportId}`, { headers: { Authorization: authToken, }, diff --git a/src/report/storage/report-storage.service.ts b/src/report/storage/report-storage.service.ts index 564fcf7..2bce71f 100644 --- a/src/report/storage/report-storage.service.ts +++ b/src/report/storage/report-storage.service.ts @@ -45,13 +45,13 @@ export class DefaultReportStorage implements ReportStorage { fetchReport(authToken: string, reportRef: Reference): Observable { return this.reportRepository.fetchReport(authToken, reportRef.id).pipe( - map((reportDoc) => { + map((report) => { return new Report( - reportDoc._id, - reportDoc.title, - reportDoc.aggregationDefinitions, + report._id, + report.title, + report.aggregationDefinitions, ).setSchema({ - fields: reportDoc.aggregationDefinitions, // todo generate actual fields here + fields: report.aggregationDefinitions, // todo generate actual fields here }); }), ); From f13d292dfa01584339867d7e243c316a4f992737 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Wed, 14 Feb 2024 12:28:27 +0100 Subject: [PATCH 04/18] chore: refactor interfaces --- .../core/couchdb-report-changes.service.ts | 6 ++---- src/report/controller/report.controller.ts | 2 +- src/report/core/report-storage.ts | 2 +- src/report/repository/report-repository.service.ts | 9 +++++++-- src/report/storage/report-storage.service.ts | 7 +++++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/report-changes/core/couchdb-report-changes.service.ts b/src/report-changes/core/couchdb-report-changes.service.ts index 284388a..26789e8 100644 --- a/src/report-changes/core/couchdb-report-changes.service.ts +++ b/src/report-changes/core/couchdb-report-changes.service.ts @@ -22,7 +22,7 @@ export class CouchdbReportChangesService implements ReportChangesService { private reportStorage: ReportStorage, private couchDbClient: CouchDbClient, ) { - // TODO: where to get databaseUrl and databaseName from? Can we centralize this ...? + // (!) TODO: where to get databaseUrl and databaseName from? Can we centralize this ...? this.couchDbClient .changes('TODO', 'app') .subscribe((changes: CouchDbChangesResponse) => { @@ -49,10 +49,8 @@ export class CouchdbReportChangesService implements ReportChangesService { async registerReportMonitoring(report: Reference) { if (!this.reportMonitors.has(report.id)) { - // TODO: reuse ReportRepository and its ReportDoc (currently not exported) here? Or duplicate some things to keep stuff isolated? - // TODO: can we centralize the auth stuff somehow? not sure where I would get this from in this context ... this.reportStorage - .fetchReport('TODO', report) + .fetchReport(report) .subscribe((report: Report | undefined) => { if (!report) { return; diff --git a/src/report/controller/report.controller.ts b/src/report/controller/report.controller.ts index 7f890c8..d4eb668 100644 --- a/src/report/controller/report.controller.ts +++ b/src/report/controller/report.controller.ts @@ -33,7 +33,7 @@ export class ReportController { @Param('reportId') reportId: string, ): Observable { return this.reportStorage - .fetchReport(token, new Reference(reportId)) + .fetchReport(new Reference(reportId), token) .pipe(switchMap((report) => this.getReportDto(report))); } private getReportDto(report: Report): Observable { diff --git a/src/report/core/report-storage.ts b/src/report/core/report-storage.ts index b969162..5150138 100644 --- a/src/report/core/report-storage.ts +++ b/src/report/core/report-storage.ts @@ -8,8 +8,8 @@ export interface ReportStorage { fetchAllReports(authToken: string, mode: string): Observable; fetchReport( - authToken: string, reportRef: Reference, + authToken?: string | undefined, ): Observable; fetchPendingCalculations(): Observable; diff --git a/src/report/repository/report-repository.service.ts b/src/report/repository/report-repository.service.ts index d4955cc..17f2c75 100644 --- a/src/report/repository/report-repository.service.ts +++ b/src/report/repository/report-repository.service.ts @@ -47,7 +47,9 @@ export class ReportRepository { this.authHeaderValue = `Basic ${authHeader}`; } - fetchReports(authToken: string): Observable { + fetchReports( + authToken: string = this.authHeaderValue, + ): Observable { return this.http .get(`${this.dbUrl}/app/_all_docs`, { params: { @@ -68,7 +70,10 @@ export class ReportRepository { ); } - fetchReport(authToken: string, reportId: string): Observable { + fetchReport( + reportId: string, + authToken: string = this.authHeaderValue, + ): Observable { return this.http .get(`${this.dbUrl}/app/${reportId}`, { headers: { diff --git a/src/report/storage/report-storage.service.ts b/src/report/storage/report-storage.service.ts index 2bce71f..f80d7da 100644 --- a/src/report/storage/report-storage.service.ts +++ b/src/report/storage/report-storage.service.ts @@ -43,8 +43,11 @@ export class DefaultReportStorage implements ReportStorage { ); } - fetchReport(authToken: string, reportRef: Reference): Observable { - return this.reportRepository.fetchReport(authToken, reportRef.id).pipe( + fetchReport( + reportRef: Reference, + authToken?: string | undefined, + ): Observable { + return this.reportRepository.fetchReport(reportRef.id, authToken).pipe( map((report) => { return new Report( report._id, From d52272a3a04d9fb363c6876d33d0454c64c6b863 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Wed, 14 Feb 2024 12:28:36 +0100 Subject: [PATCH 05/18] chore: refactor interfaces --- src/report/controller/report-calculation.controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/report/controller/report-calculation.controller.ts b/src/report/controller/report-calculation.controller.ts index 5f72bf2..2f1558b 100644 --- a/src/report/controller/report-calculation.controller.ts +++ b/src/report/controller/report-calculation.controller.ts @@ -22,7 +22,7 @@ export class ReportCalculationController { @Headers('Authorization') token: string, @Param('reportId') reportId: string, ): Observable { - return this.reportStorage.fetchReport(token, new Reference(reportId)).pipe( + return this.reportStorage.fetchReport(new Reference(reportId), token).pipe( switchMap((value) => { if (!value) { throw new NotFoundException(); @@ -64,7 +64,7 @@ export class ReportCalculationController { } return this.reportStorage - .fetchReport(token, new Reference(calculation.report.id)) + .fetchReport(new Reference(calculation.report.id), token) .pipe( map((report) => { if (!report) { From 56199d877f652b8cf2135b9f4d00d24142fba19f Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Wed, 14 Feb 2024 15:59:25 +0100 Subject: [PATCH 06/18] get module imports and dependencies aligned so that system starts --- .env | 4 +- src/couchdb/couch-db-client.service.ts | 13 ++-- src/notification/notification.module.ts | 1 + .../core/couchdb-report-changes.service.ts | 13 ++-- .../core/report-changes.service.ts | 1 - src/report-changes/report-changes.module.ts | 13 +++- ...couchdb-changes-repository.service.spec.ts | 33 ++++++++++ .../couchdb-changes-repository.service.ts | 60 +++++++++++++++++++ src/report/controller/report.controller.ts | 3 +- src/report/report.module.ts | 1 + 10 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 src/report-changes/repository/couchdb-changes-repository.service.spec.ts create mode 100644 src/report-changes/repository/couchdb-changes-repository.service.ts diff --git a/.env b/.env index 462418b..5e64c44 100644 --- a/.env +++ b/.env @@ -1,7 +1,9 @@ SENTRY_DSN= PORT= DATABASE_URL=http://127.0.0.1:5984 -DATABASE_ADMIN=admin +DATABASE_USER=admin DATABASE_PASSWORD=admin QUERY_URL=http://127.0.0.1:4984 SCHEMA_CONFIG_ID=_design/sqlite:config +REPORT_DATABASE_URL=http://127.0.0.1:5984 +REPORT_DATABASE_NAME=app \ No newline at end of file diff --git a/src/couchdb/couch-db-client.service.ts b/src/couchdb/couch-db-client.service.ts index 7d1699b..1bb1483 100644 --- a/src/couchdb/couch-db-client.service.ts +++ b/src/couchdb/couch-db-client.service.ts @@ -2,14 +2,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { catchError, map, Observable, of, switchMap } from 'rxjs'; import { HttpService } from '@nestjs/axios'; import { AxiosHeaders } from 'axios'; -import { CouchDbChangesResponse } from "./dtos"; +import { CouchDbChangesResponse } from './dtos'; -@Injectable() +@Injectable({}) export class CouchDbClient { private readonly logger = new Logger(CouchDbClient.name); - constructor(private httpService: HttpService) { - } + constructor(private httpService: HttpService) {} headDatabaseDocument( databaseUrl: string, @@ -121,14 +120,16 @@ export class CouchDbClient { this.logger.debug(err); } - changes( databaseUrl: string, databaseName: string, config?: any, ): Observable { return this.httpService - .get(`${databaseUrl}/${databaseName}/_changes`, config) + .get( + `${databaseUrl}/${databaseName}/_changes`, + config, + ) .pipe( map((response) => { return response.data; diff --git a/src/notification/notification.module.ts b/src/notification/notification.module.ts index 815726c..bafb43b 100644 --- a/src/notification/notification.module.ts +++ b/src/notification/notification.module.ts @@ -3,5 +3,6 @@ import { NotificationService } from './core/notification.service'; @Module({ providers: [NotificationService], + exports: [NotificationService], }) export class NotificationModule {} diff --git a/src/report-changes/core/couchdb-report-changes.service.ts b/src/report-changes/core/couchdb-report-changes.service.ts index 26789e8..43e7f0a 100644 --- a/src/report-changes/core/couchdb-report-changes.service.ts +++ b/src/report-changes/core/couchdb-report-changes.service.ts @@ -4,14 +4,14 @@ import { NotificationService } from '../../notification/core/notification.servic import { Reference } from '../../domain/reference'; import { ReportDataChangeEvent } from '../../domain/report-data-change-event'; import { ReportCalculation } from '../../domain/report-calculation'; -import { CouchDbClient } from '../../couchdb/couch-db-client.service'; import { CouchDbChangeResult, CouchDbChangesResponse, } from '../../couchdb/dtos'; import { Report } from '../../domain/report'; import { ReportChangesService } from './report-changes.service'; -import { ReportStorage } from '../../report/core/report-storage'; +import { CouchdbChangesRepositoryService } from '../repository/couchdb-changes-repository.service'; +import { DefaultReportStorage } from '../../report/storage/report-storage.service'; @Injectable() export class CouchdbReportChangesService implements ReportChangesService { @@ -19,12 +19,11 @@ export class CouchdbReportChangesService implements ReportChangesService { constructor( private notificationService: NotificationService, - private reportStorage: ReportStorage, - private couchDbClient: CouchDbClient, + private reportStorage: DefaultReportStorage, + private couchdbChangesRepository: CouchdbChangesRepositoryService, ) { - // (!) TODO: where to get databaseUrl and databaseName from? Can we centralize this ...? - this.couchDbClient - .changes('TODO', 'app') + this.couchdbChangesRepository + .fetchChanges() .subscribe((changes: CouchDbChangesResponse) => { // TODO: ensure continued fetching until all changes done // TODO: collect a batch of changes for a while before checking? diff --git a/src/report-changes/core/report-changes.service.ts b/src/report-changes/core/report-changes.service.ts index 74916b2..fe04cd1 100644 --- a/src/report-changes/core/report-changes.service.ts +++ b/src/report-changes/core/report-changes.service.ts @@ -1,5 +1,4 @@ /** * Monitor all changes to the application database and check if they affect any report's results. */ - export interface ReportChangesService {} diff --git a/src/report-changes/report-changes.module.ts b/src/report-changes/report-changes.module.ts index c45cc44..c698108 100644 --- a/src/report-changes/report-changes.module.ts +++ b/src/report-changes/report-changes.module.ts @@ -1,7 +1,18 @@ import { Module } from '@nestjs/common'; import { CouchdbReportChangesService } from './core/couchdb-report-changes.service'; +import { CouchdbChangesRepositoryService } from './repository/couchdb-changes-repository.service'; +import { NotificationModule } from '../notification/notification.module'; +import { ReportModule } from '../report/report.module'; +import { CouchDbClient } from '../couchdb/couch-db-client.service'; +import { HttpModule } from '@nestjs/axios'; @Module({ - providers: [CouchdbReportChangesService], + imports: [NotificationModule, ReportModule, HttpModule], + providers: [ + CouchdbReportChangesService, + CouchdbChangesRepositoryService, + CouchDbClient, // TODO: pack this into a CouchDbModule together with HttpModule import etc. + ], + exports: [CouchdbReportChangesService], }) export class ReportChangesModule {} diff --git a/src/report-changes/repository/couchdb-changes-repository.service.spec.ts b/src/report-changes/repository/couchdb-changes-repository.service.spec.ts new file mode 100644 index 0000000..e798329 --- /dev/null +++ b/src/report-changes/repository/couchdb-changes-repository.service.spec.ts @@ -0,0 +1,33 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { CouchdbChangesRepositoryService } from './couchdb-changes-repository.service'; +import { HttpModule } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; + +describe('CouchdbChangesRepositoryService', () => { + let service: CouchdbChangesRepositoryService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [HttpModule], + providers: [ + CouchdbChangesRepositoryService, + { + provide: ConfigService, + useValue: { + getOrThrow: jest.fn((key) => { + return 'foo'; + }), + }, + }, + ], + }).compile(); + + service = module.get( + CouchdbChangesRepositoryService, + ); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/report-changes/repository/couchdb-changes-repository.service.ts b/src/report-changes/repository/couchdb-changes-repository.service.ts new file mode 100644 index 0000000..3e1fbf9 --- /dev/null +++ b/src/report-changes/repository/couchdb-changes-repository.service.ts @@ -0,0 +1,60 @@ +import { + ForbiddenException, + Injectable, + NotFoundException, + UnauthorizedException, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { catchError, Observable } from 'rxjs'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { CouchDbChangesResponse } from '../../couchdb/dtos'; + +@Injectable() +export class CouchdbChangesRepositoryService { + // TODO: centralize this config by refactoring couchdbClient and providing configured clients through DI + // TODO: check if this is the correct db for our changes from app + private dbUrl: string = this.configService.getOrThrow('DATABASE_URL'); + private databaseName: string = 'app'; // TODO: move to config and clean up .env, clarifying different DBs there + private databaseUser: string = this.configService.getOrThrow('DATABASE_USER'); + private databasePassword: string = + this.configService.getOrThrow('DATABASE_PASSWORD'); + + private authHeaderValue: string; + + constructor( + private couchdbClient: CouchDbClient, + private configService: ConfigService, + ) { + const authHeader = Buffer.from( + `${this.databaseUser}:${this.databasePassword}`, + ).toString('base64'); + this.authHeaderValue = `Basic ${authHeader}`; + } + + fetchChanges(): Observable { + return this.couchdbClient + .changes(this.dbUrl, this.databaseName, { + headers: { + Authorization: this.authHeaderValue, + }, + }) + .pipe( + catchError((err, caught) => { + this.handleError(err); + throw caught; + }), + ); + } + + private handleError(err: any) { + if (err.response.status === 401) { + throw new UnauthorizedException(); + } + if (err.response.status === 403) { + throw new ForbiddenException(); + } + if (err.response.status === 404) { + throw new NotFoundException(); + } + } +} diff --git a/src/report/controller/report.controller.ts b/src/report/controller/report.controller.ts index d4eb668..2f8a47d 100644 --- a/src/report/controller/report.controller.ts +++ b/src/report/controller/report.controller.ts @@ -34,8 +34,9 @@ export class ReportController { ): Observable { return this.reportStorage .fetchReport(new Reference(reportId), token) - .pipe(switchMap((report) => this.getReportDto(report))); + .pipe(switchMap((report) => this.getReportDto(report as any))); // TODO: fix for undefined report } + private getReportDto(report: Report): Observable { return this.reportStorage .isCalculationOngoing(new Reference(report.id)) diff --git a/src/report/report.module.ts b/src/report/report.module.ts index 53cc2e0..5a68c0f 100644 --- a/src/report/report.module.ts +++ b/src/report/report.module.ts @@ -22,5 +22,6 @@ import { CouchDbClient } from '../couchdb/couch-db-client.service'; SqsReportCalculator, CouchDbClient, ], + exports: [DefaultReportStorage], }) export class ReportModule {} From 9b75f60a9a5c170a32bc4c13a8ee526ffa5d288f Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Wed, 14 Feb 2024 16:57:20 +0100 Subject: [PATCH 07/18] docs: align OpenAPI tags with module URL paths --- docs/api-specs/reporting-api-v1.yaml | 44 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/api-specs/reporting-api-v1.yaml b/docs/api-specs/reporting-api-v1.yaml index 808121b..a733a03 100644 --- a/docs/api-specs/reporting-api-v1.yaml +++ b/docs/api-specs/reporting-api-v1.yaml @@ -13,9 +13,9 @@ servers: default: dev description: Customer ID assigned by the service provider tags: - - name: report + - name: reporting description: Access reports and their results and trigger one-time report calculations. - - name: notification + - name: notifications description: Subscribe to continuous notification events whenever a report's result data changes due to users changing records in the Aam Digital application. paths: @@ -23,9 +23,9 @@ paths: get: security: - development: - - reporting_read + - reporting_read tags: - - report + - reporting summary: Return list of available Reports responses: 200: @@ -47,9 +47,9 @@ paths: get: security: - development: - - reporting_read + - reporting_read tags: - - report + - reporting summary: Return report metadata by ID parameters: - in: path @@ -81,9 +81,9 @@ paths: get: security: - development: - - reporting_read + - reporting_read tags: - - report + - reporting summary: Return all report calculations for a report parameters: - in: path @@ -114,9 +114,9 @@ paths: post: security: - development: - - reporting_write + - reporting_write tags: - - report + - reporting summary: Trigger a new report calculation run. description: Trigger a new report calculation run. Check status of the asynchronous calculation via the /report-calculation endpoint As an alternative to triggering a single report calculation, you can subscribe to receive @@ -162,9 +162,9 @@ paths: get: security: - development: - - reporting_read + - reporting_read tags: - - report + - reporting summary: Return metadata for a report calculation parameters: - in: path @@ -198,9 +198,9 @@ paths: get: security: - development: - - reporting_read + - reporting_read tags: - - report + - reporting summary: Fetch actual report data for a specific calculation parameters: - in: path @@ -237,9 +237,9 @@ paths: get: security: - development: - - reporting_read + - reporting_read tags: - - notification + - notifications summary: Return list of existing Webhooks for your authorized client responses: 200: @@ -267,9 +267,9 @@ paths: get: security: - development: - - reporting_read + - reporting_read tags: - - notification + - notifications summary: Return a specific Webhook parameters: - in: path @@ -307,9 +307,9 @@ paths: post: security: - development: - - reporting_write + - reporting_write tags: - - notification + - notifications summary: Subscribe to events for a specific Report description: If you subscribe to a reportId, Aam Digital will continuously monitor the changes users make in the application and check if they affect the results calculated by that report. @@ -356,9 +356,9 @@ paths: delete: security: - development: - - reporting_write + - reporting_write tags: - - notification + - notifications summary: Remove event subscription for a specific Report parameters: - in: path From 71332f3ec03129316889310da7ab998a203bb538 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Wed, 14 Feb 2024 17:49:51 +0100 Subject: [PATCH 08/18] docs(notifications): add missing POST endpoint to register webhook --- docs/api-specs/reporting-api-v1.yaml | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/api-specs/reporting-api-v1.yaml b/docs/api-specs/reporting-api-v1.yaml index a733a03..5173448 100644 --- a/docs/api-specs/reporting-api-v1.yaml +++ b/docs/api-specs/reporting-api-v1.yaml @@ -262,6 +262,37 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + post: + security: + - development: + - reporting_write + tags: + - notifications + summary: Subscribe to events for a specific Report + description: Register a webhook to be called for one or more notification subscriptions. To receive events, make calls to your webhookId's /subscribe endpoint. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/WebhookConfiguration' + responses: + 200: + description: Webhook registered successfully, you can now register subscriptions + content: + application/json: + schema: + type: object + properties: + id: + type: string + format: uuid + 401: + description: If no valid access token + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /webhook/{webhookId}: get: From 287f15b0b05d9ea3341ccf2351d1dcfaa2ede8a9 Mon Sep 17 00:00:00 2001 From: Sebastian Leidig Date: Wed, 14 Feb 2024 17:50:32 +0100 Subject: [PATCH 09/18] feat(notifications): add basic controllers for webhook endpoints --- src/notification/controller/dtos.ts | 14 +++ .../controller/webhook.controller.spec.ts | 18 ++++ .../controller/webhook.controller.ts | 98 +++++++++++++++++++ src/notification/core/notification.service.ts | 14 +++ src/notification/core/webhook.ts | 11 +++ src/notification/notification.module.ts | 5 +- .../storage/webhook-storage.service.spec.ts | 18 ++++ .../storage/webhook-storage.service.ts | 27 +++++ 8 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 src/notification/controller/dtos.ts create mode 100644 src/notification/controller/webhook.controller.spec.ts create mode 100644 src/notification/controller/webhook.controller.ts create mode 100644 src/notification/core/webhook.ts create mode 100644 src/notification/storage/webhook-storage.service.spec.ts create mode 100644 src/notification/storage/webhook-storage.service.ts diff --git a/src/notification/controller/dtos.ts b/src/notification/controller/dtos.ts new file mode 100644 index 0000000..35b39d2 --- /dev/null +++ b/src/notification/controller/dtos.ts @@ -0,0 +1,14 @@ +export interface WebhookDto {} + +export interface WebhookConfigurationDto { + name: string; + method: 'GET' | 'POST'; + targetUrl: string; + authenticationType: 'API_KEY'; + authentication: ApiKeyAuthConfig; +} + +export interface ApiKeyAuthConfig { + key: string; + headerName: string; +} diff --git a/src/notification/controller/webhook.controller.spec.ts b/src/notification/controller/webhook.controller.spec.ts new file mode 100644 index 0000000..959fe61 --- /dev/null +++ b/src/notification/controller/webhook.controller.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { WebhookController } from './webhook.controller'; + +describe('WebhookController', () => { + let controller: WebhookController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [WebhookController], + }).compile(); + + controller = module.get(WebhookController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/src/notification/controller/webhook.controller.ts b/src/notification/controller/webhook.controller.ts new file mode 100644 index 0000000..4a720cd --- /dev/null +++ b/src/notification/controller/webhook.controller.ts @@ -0,0 +1,98 @@ +import { + Body, + Controller, + Delete, + Get, + Headers, + Param, + Post, +} from '@nestjs/common'; +import { defaultIfEmpty, map, Observable, zipAll } from 'rxjs'; +import { Reference } from '../../domain/reference'; +import { WebhookStorageService } from '../storage/webhook-storage.service'; +import { Webhook } from '../core/webhook'; +import { NotificationService } from '../core/notification.service'; +import { WebhookConfigurationDto, WebhookDto } from './dtos'; + +@Controller('/api/v1/notifications/webhook') +export class WebhookController { + constructor( + private webhookStorage: WebhookStorageService, + private notificationService: NotificationService, + ) {} + + @Get() + fetchWebhooksOfUser( + @Headers('Authorization') token: string, + ): Observable { + return this.webhookStorage.fetchAllWebhooks(token).pipe( + map((webhooks) => webhooks.map((webhook) => this.getWebhookDto(webhook))), + zipAll(), + defaultIfEmpty([]), + ); + } + + @Get('/:webhookId') + fetchWebhook( + @Headers('Authorization') token: string, + @Param('webhookId') webhookId: string, + ): Observable { + return this.webhookStorage.fetchWebhook(new Reference(webhookId)).pipe( + // TODO: check auth? + // TODO: map to 404 if undefined + map((webhook) => this.getWebhookDto(webhook as any)), + ); + } + + @Post() + createWebhook( + @Headers('Authorization') token: string, + @Body() requestBody: WebhookConfigurationDto, + ): Observable { + return this.webhookStorage.createWebhook(requestBody).pipe( + // TODO: check auth? + // TODO: map errors to response codes + map((webhookRef: Reference) => webhookRef.id), + ); + } + + @Post('/:webhookId/subscribe/report/:reportId') + subscribeReportNotifications( + @Headers('Authorization') token: string, + @Param('webhookId') webhookId: string, + @Param('reportId') reportId: string, + ): Observable { + return this.notificationService + .registerForReportEvents( + new Reference(webhookId), + new Reference(reportId), + ) + .pipe + // TODO: check auth? + // TODO: map errors to response codes + // TODO: map to 200 Response without body (otherwise service throws error) + (); + } + + @Delete('/:webhookId/subscribe/report/:reportId') + unsubscribeReportNotifications( + @Headers('Authorization') token: string, + @Param('webhookId') webhookId: string, + @Param('reportId') reportId: string, + ): Observable { + return this.notificationService + .unregisterForReportEvents( + new Reference(webhookId), + new Reference(reportId), + ) + .pipe + // TODO: check auth? + // TODO: map errors to response codes + // TODO: map to 200 Response without body (otherwise service throws error) + (); + } + + private getWebhookDto(webhook: Webhook): WebhookDto { + return webhook; + } +} diff --git a/src/notification/core/notification.service.ts b/src/notification/core/notification.service.ts index d5b2a17..4a852d7 100644 --- a/src/notification/core/notification.service.ts +++ b/src/notification/core/notification.service.ts @@ -22,4 +22,18 @@ export class NotificationService { * Trigger a core event for the given report to any active subscribers. */ triggerNotification(event: ReportDataChangeEvent): void {} + + registerForReportEvents( + webhook: Reference, + report: Reference, + ): Observable { + return of(); + } + + unregisterForReportEvents( + webhook: Reference, + report: Reference, + ): Observable { + return of(); + } } diff --git a/src/notification/core/webhook.ts b/src/notification/core/webhook.ts new file mode 100644 index 0000000..2fc384d --- /dev/null +++ b/src/notification/core/webhook.ts @@ -0,0 +1,11 @@ +import { Reference } from '../../domain/reference'; +import { WebhookConfigurationDto } from '../controller/dtos'; + +export interface Webhook extends WebhookConfiguration { + id: string; + name: string; // TODO: why name? + + reportSubscriptions: Reference[]; +} + +export type WebhookConfiguration = WebhookConfigurationDto; diff --git a/src/notification/notification.module.ts b/src/notification/notification.module.ts index bafb43b..630a972 100644 --- a/src/notification/notification.module.ts +++ b/src/notification/notification.module.ts @@ -1,8 +1,11 @@ import { Module } from '@nestjs/common'; import { NotificationService } from './core/notification.service'; +import { WebhookStorageService } from './storage/webhook-storage.service'; +import { WebhookController } from './controller/webhook.controller'; @Module({ - providers: [NotificationService], + controllers: [WebhookController], + providers: [NotificationService, WebhookStorageService], exports: [NotificationService], }) export class NotificationModule {} diff --git a/src/notification/storage/webhook-storage.service.spec.ts b/src/notification/storage/webhook-storage.service.spec.ts new file mode 100644 index 0000000..4478c71 --- /dev/null +++ b/src/notification/storage/webhook-storage.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { WebhookStorageService } from './webhook-storage.service'; + +describe('WebhookStorageService', () => { + let service: WebhookStorageService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [WebhookStorageService], + }).compile(); + + service = module.get(WebhookStorageService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/notification/storage/webhook-storage.service.ts b/src/notification/storage/webhook-storage.service.ts new file mode 100644 index 0000000..9b35b50 --- /dev/null +++ b/src/notification/storage/webhook-storage.service.ts @@ -0,0 +1,27 @@ +import { Injectable } from '@nestjs/common'; +import { Observable, of } from 'rxjs'; +import { Webhook, WebhookConfiguration } from '../core/webhook'; +import { Reference } from '../../domain/reference'; + +@Injectable() +export class WebhookStorageService { + /** + * Get all registered webhooks subscribe by the user authenticated with the given token + * @param token + */ + fetchAllWebhooks(token: string): Observable { + return of([]); + } + + fetchWebhook(webhook: Reference): Observable { + return of(undefined); + } + + /** + * Creates a new webhook with the given configuration, stores it and returns a reference to the new webhook. + * @param webhookConfig + */ + createWebhook(webhookConfig: WebhookConfiguration): Observable { + return of(new Reference('new-webhook-id')); + } +} From 00f86c32f367ad8903a51d82daed033a27d2998c Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Sat, 17 Feb 2024 13:32:30 +0100 Subject: [PATCH 10/18] feat: implement missing webhook interfaces --- .env | 13 +- README.md | 2 +- package-lock.json | 12 -- src/config/app.yaml | 5 + src/couchdb/couch-db-client.service.ts | 110 +++++++-------- src/couchdb/dtos.ts | 6 + src/crypto/core/crypto.service.ts | 47 +++++++ src/crypto/crypto/crypto.module.ts | 17 +++ src/crypto/di/crypto-configuration.ts | 16 +++ src/domain/report-data.ts | 4 +- src/notification/controller/dtos.ts | 24 +++- .../controller/webhook.controller.spec.ts | 4 +- .../controller/webhook.controller.ts | 88 ++++++------ src/notification/core/notification.service.ts | 104 +++++++++++++- src/notification/core/webhook.ts | 11 -- .../di/notification-configuration.ts | 43 ++++++ src/notification/domain/webhook.ts | 31 ++++ src/notification/notification.module.ts | 17 ++- .../webhook-repository.service.spec.ts | 18 +++ .../repository/webhook-repository.service.ts | 71 ++++++++++ .../storage/webhook-storage.service.spec.ts | 8 +- .../storage/webhook-storage.service.ts | 133 ++++++++++++++++-- .../core/report-changes.service.ts | 1 - src/report/di/couchdb-sqs-configuration.ts | 12 +- src/report/report.module.ts | 4 +- .../report-calculation-repository.service.ts | 86 ++++++----- .../repository/report-repository.service.ts | 7 +- .../report-calculation-processor.service.ts | 2 +- 28 files changed, 686 insertions(+), 210 deletions(-) create mode 100644 src/crypto/core/crypto.service.ts create mode 100644 src/crypto/crypto/crypto.module.ts create mode 100644 src/crypto/di/crypto-configuration.ts delete mode 100644 src/notification/core/webhook.ts create mode 100644 src/notification/di/notification-configuration.ts create mode 100644 src/notification/domain/webhook.ts create mode 100644 src/notification/repository/webhook-repository.service.spec.ts create mode 100644 src/notification/repository/webhook-repository.service.ts diff --git a/.env b/.env index 5e64c44..f709b92 100644 --- a/.env +++ b/.env @@ -2,8 +2,17 @@ SENTRY_DSN= PORT= DATABASE_URL=http://127.0.0.1:5984 DATABASE_USER=admin -DATABASE_PASSWORD=admin +DATABASE_PASSWORD=docker QUERY_URL=http://127.0.0.1:4984 SCHEMA_CONFIG_ID=_design/sqlite:config + REPORT_DATABASE_URL=http://127.0.0.1:5984 -REPORT_DATABASE_NAME=app \ No newline at end of file +REPORT_DATABASE_NAME=report-calculation + +COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_USER=admin +COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker + +NOTIFICATION_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_USER=admin +NOTIFICATION_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker + +CRYPTO_ENCRYPTION_SECRET=super-duper-secret diff --git a/README.md b/README.md index d409c07..1f5378f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Query Backend +# Query Back'end This service allows to run SQL queries on the database. In particular, this service allows users with limited permissions to see reports of aggregated statistics across all data (e.g. a supervisor could analyse reports without having access to possibly confidential details of participants or notes). diff --git a/package-lock.json b/package-lock.json index 43ddb6b..01873dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,6 @@ "@nestjs/schedule": "4.0.0", "@ntegral/nestjs-sentry": "^4.0.0", "@sentry/node": "^7.38.0", - "flat": "6.0.1", "js-yaml": "4.1.0", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", @@ -4545,17 +4544,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/flat": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/flat/-/flat-6.0.1.tgz", - "integrity": "sha512-/3FfIa8mbrg3xE7+wAhWeV+bd7L2Mof+xtZb5dRDKZ+wDvYJK4WDYeIOuOhre5Yv5aQObZrlbRmk3RTSiuQBtw==", - "bin": { - "flat": "cli.js" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/flat-cache": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.1.1.tgz", diff --git a/src/config/app.yaml b/src/config/app.yaml index 42e0b0b..e506b77 100644 --- a/src/config/app.yaml +++ b/src/config/app.yaml @@ -1,2 +1,7 @@ COUCH_SQS_CLIENT_CONFIG: BASE_URL: http://localhost:4984 + +NOTIFICATION: + COUCH_DB_CLIENT_CONFIG: + BASE_URL: http://localhost:5984 + TARGET_DATABASE: notification-webhook diff --git a/src/couchdb/couch-db-client.service.ts b/src/couchdb/couch-db-client.service.ts index 4833ff8..f261e2e 100644 --- a/src/couchdb/couch-db-client.service.ts +++ b/src/couchdb/couch-db-client.service.ts @@ -4,21 +4,33 @@ import { HttpService } from '@nestjs/axios'; import { AxiosHeaders } from 'axios'; import { CouchDbChangesResponse } from './dtos'; +export class CouchDbClientConfig { + BASE_URL = ''; + TARGET_DATABASE = ''; + BASIC_AUTH_USER = ''; + BASIC_AUTH_PASSWORD = ''; +} + @Injectable() export class CouchDbClient { private readonly logger = new Logger(CouchDbClient.name); constructor(private httpService: HttpService) {} - headDatabaseDocument( + changes( databaseUrl: string, databaseName: string, - documentId: string, config?: any, - ) { + ): Observable { return this.httpService - .head(`${databaseUrl}/${databaseName}/${documentId}`, config) + .get( + `${databaseUrl}/${databaseName}/_changes`, + config, + ) .pipe( + map((response) => { + return response.data; + }), catchError((err) => { this.handleError(err); throw err; @@ -26,14 +38,23 @@ export class CouchDbClient { ); } - getDatabaseDocument( - databaseUrl: string, - databaseName: string, - documentId: string, - config?: any, - ): Observable { + headDatabaseDocument(request: { documentId: string; config?: any }) { + return this.httpService.head(`${request.documentId}`, request.config).pipe( + catchError((err) => { + if (err.response.status !== 404) { + this.handleError(err); + } + throw err; + }), + ); + } + + getDatabaseDocument(request: { + documentId: string; + config?: any; + }): Observable { return this.httpService - .get(`${databaseUrl}/${databaseName}/${documentId}`, config) + .get(`${request.documentId}`, request.config) .pipe( map((response) => { return response.data; @@ -64,21 +85,25 @@ export class CouchDbClient { ); } - putDatabaseDocument( - databaseUrl: string, - databaseName: string, - documentId: string, - body: any, - config?: any, - ): Observable { - return this.latestRef(databaseUrl, databaseName, documentId, config).pipe( + putDatabaseDocument(request: { + documentId: string; + body: any; + config: any; + }): Observable { + return this.latestRef({ + documentId: request.documentId, + config: request.config, + }).pipe( switchMap((rev) => { if (rev) { - config.headers['If-Match'] = rev; + if (!request.config.headers) { + request.config.headers = {}; + } + request.config.headers['If-Match'] = rev; } return this.httpService - .put(`${databaseUrl}/${databaseName}/${documentId}`, body, config) + .put(request.documentId, request.body, request.config) .pipe( map((response) => { return response.data; @@ -92,18 +117,14 @@ export class CouchDbClient { ); } - private latestRef( - databaseUrl: string, - databaseName: string, - documentId: string, - config?: any, - ): Observable { - return this.headDatabaseDocument( - databaseUrl, - databaseName, - documentId, - config, - ).pipe( + private latestRef(request: { + documentId: string; + config?: any; + }): Observable { + return this.headDatabaseDocument({ + documentId: request.documentId, + config: request.config, + }).pipe( map((response): string | undefined => { const headers = response.headers; if (headers instanceof AxiosHeaders && headers.has('etag')) { @@ -117,27 +138,6 @@ export class CouchDbClient { } private handleError(err: any) { - this.logger.debug(err); - } - - changes( - databaseUrl: string, - databaseName: string, - config?: any, - ): Observable { - return this.httpService - .get( - `${databaseUrl}/${databaseName}/_changes`, - config, - ) - .pipe( - map((response) => { - return response.data; - }), - catchError((err) => { - this.handleError(err); - throw err; - }), - ); + this.logger.error(err); } } diff --git a/src/couchdb/dtos.ts b/src/couchdb/dtos.ts index 0d282c1..08a64b2 100644 --- a/src/couchdb/dtos.ts +++ b/src/couchdb/dtos.ts @@ -7,6 +7,12 @@ export interface CouchDbRow { doc: T; } +export interface CouchDbRows { + total_rows: number; + offset: number; + rows: T[]; +} + export class DocSuccess { ok: boolean; id: string; diff --git a/src/crypto/core/crypto.service.ts b/src/crypto/core/crypto.service.ts new file mode 100644 index 0000000..bc4e262 --- /dev/null +++ b/src/crypto/core/crypto.service.ts @@ -0,0 +1,47 @@ +import * as crypto from 'crypto'; + +export class CryptoConfig { + ENCRYPTION_SECRET = ''; +} + +export class CryptoService { + constructor(private config: CryptoConfig) {} + + decrypt(data: { iv: string; data: string }): string { + const encryptedTextBuffer = Buffer.from(data.data, 'hex'); + const decipher = crypto.createDecipheriv( + 'aes-256-cbc', + this.getHash(this.config.ENCRYPTION_SECRET), + Buffer.from(data.iv, 'hex'), + ); + + let decrypted = decipher.update(encryptedTextBuffer); + decrypted = Buffer.concat([decrypted, decipher.final()]); + + return decrypted.toString(); + } + + encrypt(text: string): { + iv: string; + data: string; + } { + const iv = crypto.randomBytes(16); + const cipher = crypto.createCipheriv( + 'aes-256-cbc', + this.getHash(this.config.ENCRYPTION_SECRET), + iv, + ); + + let encryptedText = cipher.update(text); + encryptedText = Buffer.concat([encryptedText, cipher.final()]); + + return { + iv: iv.toString('hex'), + data: encryptedText.toString('hex'), + }; + } + + private getHash(text: string): Buffer { + return crypto.createHash('sha256').update(text).digest(); + } +} diff --git a/src/crypto/crypto/crypto.module.ts b/src/crypto/crypto/crypto.module.ts new file mode 100644 index 0000000..b0adf85 --- /dev/null +++ b/src/crypto/crypto/crypto.module.ts @@ -0,0 +1,17 @@ +import { Module } from '@nestjs/common'; +import { CryptoService } from '../core/crypto.service'; +import { CryptoServiceFactory } from '../di/crypto-configuration'; +import { ConfigModule, ConfigService } from '@nestjs/config'; + +@Module({ + imports: [ConfigModule], + providers: [ + { + provide: CryptoService, + useFactory: CryptoServiceFactory, + inject: [ConfigService], + }, + ], + exports: [CryptoService], +}) +export class CryptoModule {} diff --git a/src/crypto/di/crypto-configuration.ts b/src/crypto/di/crypto-configuration.ts new file mode 100644 index 0000000..f7d3e63 --- /dev/null +++ b/src/crypto/di/crypto-configuration.ts @@ -0,0 +1,16 @@ +import { ConfigService } from '@nestjs/config'; +import { CryptoConfig, CryptoService } from '../core/crypto.service'; + +export const CryptoServiceFactory = ( + configService: ConfigService, +): CryptoService => { + const CONFIG_BASE = 'CRYPTO_'; + + const config: CryptoConfig = { + ENCRYPTION_SECRET: configService.getOrThrow( + CONFIG_BASE + 'ENCRYPTION_SECRET', + ), + }; + + return new CryptoService(config); +}; diff --git a/src/domain/report-data.ts b/src/domain/report-data.ts index 89c66b0..027f4da 100644 --- a/src/domain/report-data.ts +++ b/src/domain/report-data.ts @@ -25,10 +25,10 @@ export class ReportData { return this; } - asHash(): string { + getDataHash(): string { return crypto .createHash('sha256') - .update(JSON.stringify(this)) + .update(JSON.stringify(this.data)) .digest('hex'); } } diff --git a/src/notification/controller/dtos.ts b/src/notification/controller/dtos.ts index 35b39d2..4292c6f 100644 --- a/src/notification/controller/dtos.ts +++ b/src/notification/controller/dtos.ts @@ -1,11 +1,23 @@ -export interface WebhookDto {} - -export interface WebhookConfigurationDto { +export interface WebhookDto { + id: string; name: string; - method: 'GET' | 'POST'; - targetUrl: string; + target: { + method: 'GET' | 'POST'; + url: string; + }; authenticationType: 'API_KEY'; - authentication: ApiKeyAuthConfig; +} + +export interface CreateWebhookDto { + label: string; + target: { + method: 'GET' | 'POST'; + url: string; + }; + authentication: { + type: 'API_KEY'; + apiKey: string; + }; } export interface ApiKeyAuthConfig { diff --git a/src/notification/controller/webhook.controller.spec.ts b/src/notification/controller/webhook.controller.spec.ts index f09d27c..e4913f9 100644 --- a/src/notification/controller/webhook.controller.spec.ts +++ b/src/notification/controller/webhook.controller.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { WebhookController } from './webhook.controller'; -import { WebhookStorageService } from '../storage/webhook-storage.service'; +import { WebhookStorage } from '../storage/webhook-storage.service'; import { NotificationService } from '../core/notification.service'; describe('WebhookController', () => { @@ -10,7 +10,7 @@ describe('WebhookController', () => { const module: TestingModule = await Test.createTestingModule({ controllers: [WebhookController], providers: [ - { provide: WebhookStorageService, useValue: {} }, + { provide: WebhookStorage, useValue: {} }, { provide: NotificationService, useValue: {} }, ], }).compile(); diff --git a/src/notification/controller/webhook.controller.ts b/src/notification/controller/webhook.controller.ts index 4a720cd..8b364ed 100644 --- a/src/notification/controller/webhook.controller.ts +++ b/src/notification/controller/webhook.controller.ts @@ -1,4 +1,5 @@ import { + BadRequestException, Body, Controller, Delete, @@ -9,15 +10,15 @@ import { } from '@nestjs/common'; import { defaultIfEmpty, map, Observable, zipAll } from 'rxjs'; import { Reference } from '../../domain/reference'; -import { WebhookStorageService } from '../storage/webhook-storage.service'; -import { Webhook } from '../core/webhook'; +import { WebhookStorage } from '../storage/webhook-storage.service'; +import { Webhook } from '../domain/webhook'; import { NotificationService } from '../core/notification.service'; -import { WebhookConfigurationDto, WebhookDto } from './dtos'; +import { CreateWebhookDto, WebhookDto } from './dtos'; -@Controller('/api/v1/notifications/webhook') +@Controller('/api/v1/reporting/webhook') export class WebhookController { constructor( - private webhookStorage: WebhookStorageService, + private webhookStorage: WebhookStorage, private notificationService: NotificationService, ) {} @@ -25,8 +26,8 @@ export class WebhookController { fetchWebhooksOfUser( @Headers('Authorization') token: string, ): Observable { - return this.webhookStorage.fetchAllWebhooks(token).pipe( - map((webhooks) => webhooks.map((webhook) => this.getWebhookDto(webhook))), + return this.webhookStorage.fetchAllWebhooks('user-token').pipe( + map((webhooks) => webhooks.map((webhook) => this.mapToDto(webhook))), zipAll(), defaultIfEmpty([]), ); @@ -39,21 +40,31 @@ export class WebhookController { ): Observable { return this.webhookStorage.fetchWebhook(new Reference(webhookId)).pipe( // TODO: check auth? - // TODO: map to 404 if undefined - map((webhook) => this.getWebhookDto(webhook as any)), + map((webhook) => { + if (!webhook) { + throw new BadRequestException(); + } + return this.mapToDto(webhook); + }), ); } @Post() createWebhook( @Headers('Authorization') token: string, - @Body() requestBody: WebhookConfigurationDto, - ): Observable { - return this.webhookStorage.createWebhook(requestBody).pipe( - // TODO: check auth? - // TODO: map errors to response codes - map((webhookRef: Reference) => webhookRef.id), - ); + @Body() requestBody: CreateWebhookDto, + ): Observable { + return this.webhookStorage + .createWebhook({ + label: requestBody.label, + target: requestBody.target, + authentication: requestBody.authentication, + }) + .pipe( + // TODO: check auth? + // TODO: map errors to response codes + map((webhookRef: Reference) => webhookRef), + ); } @Post('/:webhookId/subscribe/report/:reportId') @@ -61,17 +72,13 @@ export class WebhookController { @Headers('Authorization') token: string, @Param('webhookId') webhookId: string, @Param('reportId') reportId: string, - ): Observable { - return this.notificationService - .registerForReportEvents( - new Reference(webhookId), - new Reference(reportId), - ) - .pipe - // TODO: check auth? - // TODO: map errors to response codes - // TODO: map to 200 Response without body (otherwise service throws error) - (); + ): Observable { + return this.notificationService.registerForReportEvents( + new Reference(webhookId), + new Reference(reportId), + ); + // TODO: check auth? + // TODO: map errors to response codes } @Delete('/:webhookId/subscribe/report/:reportId') @@ -79,20 +86,21 @@ export class WebhookController { @Headers('Authorization') token: string, @Param('webhookId') webhookId: string, @Param('reportId') reportId: string, - ): Observable { - return this.notificationService - .unregisterForReportEvents( - new Reference(webhookId), - new Reference(reportId), - ) - .pipe - // TODO: check auth? - // TODO: map errors to response codes - // TODO: map to 200 Response without body (otherwise service throws error) - (); + ): Observable { + return this.notificationService.unregisterForReportEvents( + new Reference(webhookId), + new Reference(reportId), + ); + // TODO: check auth? + // TODO: map errors to response codes } - private getWebhookDto(webhook: Webhook): WebhookDto { - return webhook; + private mapToDto(webhook: Webhook): WebhookDto { + return { + id: webhook.id, + name: webhook.label, + target: webhook.target, + authenticationType: webhook.authentication.type, + }; } } diff --git a/src/notification/core/notification.service.ts b/src/notification/core/notification.service.ts index 4a852d7..6adad2f 100644 --- a/src/notification/core/notification.service.ts +++ b/src/notification/core/notification.service.ts @@ -1,39 +1,129 @@ import { Injectable } from '@nestjs/common'; -import { Observable, of } from 'rxjs'; +import { + BehaviorSubject, + catchError, + map, + mergeMap, + Observable, + switchMap, + zipAll, +} from 'rxjs'; import { Reference } from '../../domain/reference'; import { ReportDataChangeEvent } from '../../domain/report-data-change-event'; +import { WebhookStorage } from '../storage/webhook-storage.service'; +import { HttpService } from '@nestjs/axios'; +import { Webhook } from '../domain/webhook'; /** * Manage core subscriptions and delivering events to subscribers. */ @Injectable() export class NotificationService { + private _activeReports: BehaviorSubject = new BehaviorSubject( + [] as Reference[], + ); + + constructor( + private webhookStorage: WebhookStorage, + private httpService: HttpService, + ) {} + /** * Get the list of reports for which notifications are subscribed by at least one client. */ activeReports(): Observable { + return this.webhookStorage.fetchAllWebhooks().pipe( + map((webhooks) => + webhooks.flatMap((webhook) => webhook.reportSubscriptions), + ), + map((reports) => this._activeReports.next(reports)), + switchMap(() => this._activeReports.asObservable()), + ); + // TODO: is this emitting the whole list every time the subscriptions change, as the name suggests? // or individual id when added (but then, how is unsubscribe tracked?) // may be easier if I can just directly get the list of currently active reports - return of([]); } /** * Trigger a core event for the given report to any active subscribers. */ - triggerNotification(event: ReportDataChangeEvent): void {} + triggerNotification(event: ReportDataChangeEvent): void { + // todo call webhook with arguments and placeholder + + this.webhookStorage + .fetchAllWebhooks() + .pipe( + map((webhooks): Webhook[] => + webhooks.filter( + (webhook) => + webhook.reportSubscriptions.findIndex( + (reportRef) => reportRef.id === event.report.id, + ) !== -1, + ), + ), + mergeMap((webhooks) => + webhooks.map((webhook) => + this.httpService + .request({ + method: webhook.target.method, + url: webhook.target.url, + headers: { + 'X-API-KEY': webhook.authentication.apiKey, + }, + timeout: 5000, + }) + .pipe( + map((response) => { + console.log('axios response', response); + }), + catchError((err) => { + console.log('could not send notification to webhook', err); + throw err; + }), + ), + ), + ), + zipAll(), + ) + .subscribe({ + next: (value) => { + console.log('webhook send'); + }, + error: (err) => { + console.log(err); + }, + complete: () => { + console.log('webhook trigger completed'); + }, + }); + } registerForReportEvents( webhook: Reference, report: Reference, - ): Observable { - return of(); + ): Observable { + return this.webhookStorage.addSubscription(webhook, report).pipe( + map(() => { + if (!this._activeReports.value.find((ref) => ref.id === report.id)) { + this._activeReports.next([...this._activeReports.value, report]); + } + return null; + }), + ); } unregisterForReportEvents( webhook: Reference, report: Reference, - ): Observable { - return of(); + ): Observable { + return this.webhookStorage.removeSubscription(webhook, report).pipe( + map(() => { + this._activeReports.next( + this._activeReports.value.filter((ref) => ref.id !== report.id), + ); + return null; + }), + ); } } diff --git a/src/notification/core/webhook.ts b/src/notification/core/webhook.ts deleted file mode 100644 index 2fc384d..0000000 --- a/src/notification/core/webhook.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Reference } from '../../domain/reference'; -import { WebhookConfigurationDto } from '../controller/dtos'; - -export interface Webhook extends WebhookConfiguration { - id: string; - name: string; // TODO: why name? - - reportSubscriptions: Reference[]; -} - -export type WebhookConfiguration = WebhookConfigurationDto; diff --git a/src/notification/di/notification-configuration.ts b/src/notification/di/notification-configuration.ts new file mode 100644 index 0000000..85c1d8f --- /dev/null +++ b/src/notification/di/notification-configuration.ts @@ -0,0 +1,43 @@ +import { HttpService } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; +import { + CouchDbClient, + CouchDbClientConfig, +} from '../../couchdb/couch-db-client.service'; +import { WebhookStorage } from '../storage/webhook-storage.service'; +import { WebhookRepository } from '../repository/webhook-repository.service'; +import { CryptoService } from '../../crypto/core/crypto.service'; +import axios from 'axios'; + +export const CouchDbClientFactory = ( + configService: ConfigService, +): CouchDbClient => { + const CONFIG_BASE = 'NOTIFICATION_COUCH_DB_CLIENT_CONFIG_'; + + const config: CouchDbClientConfig = { + BASE_URL: configService.getOrThrow(CONFIG_BASE + 'BASE_URL'), + TARGET_DATABASE: configService.getOrThrow(CONFIG_BASE + 'TARGET_DATABASE'), + BASIC_AUTH_USER: configService.getOrThrow(CONFIG_BASE + 'BASIC_AUTH_USER'), + BASIC_AUTH_PASSWORD: configService.getOrThrow( + CONFIG_BASE + 'BASIC_AUTH_PASSWORD', + ), + }; + + const axiosInstance = axios.create(); + + axiosInstance.defaults.baseURL = `${config.BASE_URL}/${config.TARGET_DATABASE}`; + axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( + `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, + ).toString('base64')}`; + + return new CouchDbClient(new HttpService(axiosInstance)); +}; + +export const WebhookStorageFactory = ( + cryptoService: CryptoService, + configService: ConfigService, +): WebhookStorage => { + const couchDbClient = CouchDbClientFactory(configService); + const webhookRepository = new WebhookRepository(couchDbClient); + return new WebhookStorage(webhookRepository, cryptoService); +}; diff --git a/src/notification/domain/webhook.ts b/src/notification/domain/webhook.ts new file mode 100644 index 0000000..95a18e2 --- /dev/null +++ b/src/notification/domain/webhook.ts @@ -0,0 +1,31 @@ +import { Reference } from '../../domain/reference'; + +export interface Webhook { + id: string; + label: string; + target: { + method: 'GET' | 'POST'; + url: string; + }; + authentication: { + type: 'API_KEY'; + apiKey: string; + }; + owner: { + type: 'USER'; // TODO: group support? + id: string; + }; + reportSubscriptions: Reference[]; +} + +export interface CreateWebhookRequest { + label: string; + target: { + method: 'GET' | 'POST'; + url: string; + }; + authentication: { + type: 'API_KEY'; + apiKey: string; + }; +} diff --git a/src/notification/notification.module.ts b/src/notification/notification.module.ts index 630a972..83fc596 100644 --- a/src/notification/notification.module.ts +++ b/src/notification/notification.module.ts @@ -1,11 +1,24 @@ import { Module } from '@nestjs/common'; import { NotificationService } from './core/notification.service'; -import { WebhookStorageService } from './storage/webhook-storage.service'; +import { WebhookStorage } from './storage/webhook-storage.service'; import { WebhookController } from './controller/webhook.controller'; +import { ConfigService } from '@nestjs/config'; +import { WebhookStorageFactory } from './di/notification-configuration'; +import { CryptoModule } from '../crypto/crypto/crypto.module'; +import { CryptoService } from '../crypto/core/crypto.service'; +import { HttpModule } from '@nestjs/axios'; @Module({ controllers: [WebhookController], - providers: [NotificationService, WebhookStorageService], + imports: [CryptoModule, HttpModule], + providers: [ + NotificationService, + { + provide: WebhookStorage, + useFactory: WebhookStorageFactory, + inject: [CryptoService, ConfigService], + }, + ], exports: [NotificationService], }) export class NotificationModule {} diff --git a/src/notification/repository/webhook-repository.service.spec.ts b/src/notification/repository/webhook-repository.service.spec.ts new file mode 100644 index 0000000..5a2bbb1 --- /dev/null +++ b/src/notification/repository/webhook-repository.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { WebhookRepository } from './webhook-repository.service'; + +describe('CouchWebhookRepositoryService', () => { + let service: WebhookRepository; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [WebhookRepository], + }).compile(); + + service = module.get(WebhookRepository); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/notification/repository/webhook-repository.service.ts b/src/notification/repository/webhook-repository.service.ts new file mode 100644 index 0000000..fbc3a7b --- /dev/null +++ b/src/notification/repository/webhook-repository.service.ts @@ -0,0 +1,71 @@ +import { NotFoundException } from '@nestjs/common'; +import { catchError, map, Observable } from 'rxjs'; +import { Reference } from '../../domain/reference'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { CouchDbRow, CouchDbRows } from '../../couchdb/dtos'; + +export interface WebhookEntity { + id: string; + label: string; + target: { + method: 'GET' | 'POST'; + url: string; + }; + authentication: { + type: 'API_KEY'; + apiKey: { + iv: string; + data: string; + }; + }; + owner: { + type: 'USER'; // TODO: group support? + id: string; + }; + reportSubscriptions: Reference[]; +} + +export class WebhookRepository { + constructor(private http: CouchDbClient) {} + + fetchAllWebhooks(): Observable { + return this.http + .getDatabaseDocument>>({ + documentId: '_all_docs', + config: { + params: { + include_docs: true, + start_key: '"Webhook"', + end_key: '"Webhook' + '\ufff0"', // ufff0 -> high value unicode character + }, + }, + }) + .pipe( + map((rows) => rows.rows), + map((row) => row.map((entity) => entity.doc)), + ); + } + + fetchWebhook(webhookRef: Reference): Observable { + return this.http + .getDatabaseDocument({ + documentId: webhookRef.id, + }) + .pipe( + catchError((err) => { + if (err.response.status === 404) { + throw new NotFoundException(); + } + throw err; + }), + ); + } + + storeWebhook(webhook: WebhookEntity): Observable { + return this.http.putDatabaseDocument({ + documentId: webhook.id, + body: webhook, + config: {}, + }); + } +} diff --git a/src/notification/storage/webhook-storage.service.spec.ts b/src/notification/storage/webhook-storage.service.spec.ts index 4478c71..45fbabd 100644 --- a/src/notification/storage/webhook-storage.service.spec.ts +++ b/src/notification/storage/webhook-storage.service.spec.ts @@ -1,15 +1,15 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { WebhookStorageService } from './webhook-storage.service'; +import { WebhookStorage } from './webhook-storage.service'; describe('WebhookStorageService', () => { - let service: WebhookStorageService; + let service: WebhookStorage; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - providers: [WebhookStorageService], + providers: [WebhookStorage], }).compile(); - service = module.get(WebhookStorageService); + service = module.get(WebhookStorage); }); it('should be defined', () => { diff --git a/src/notification/storage/webhook-storage.service.ts b/src/notification/storage/webhook-storage.service.ts index 9b35b50..0eaf21a 100644 --- a/src/notification/storage/webhook-storage.service.ts +++ b/src/notification/storage/webhook-storage.service.ts @@ -1,27 +1,136 @@ -import { Injectable } from '@nestjs/common'; -import { Observable, of } from 'rxjs'; -import { Webhook, WebhookConfiguration } from '../core/webhook'; +import { map, Observable, switchMap } from 'rxjs'; +import { CreateWebhookRequest, Webhook } from '../domain/webhook'; import { Reference } from '../../domain/reference'; +import { + WebhookEntity, + WebhookRepository, +} from '../repository/webhook-repository.service'; +import { v4 as uuidv4 } from 'uuid'; +import { CryptoService } from '../../crypto/core/crypto.service'; +import { NotFoundException } from '@nestjs/common'; + +export class WebhookStorage { + constructor( + private webhookRepository: WebhookRepository, + private cryptoService: CryptoService, + ) {} + + addSubscription( + webhookRef: Reference, + entityRef: Reference, + ): Observable { + return this.fetchWebhook(webhookRef).pipe( + map((webhook): Webhook => { + if (!webhook) { + throw new NotFoundException(); + } + + if ( + !webhook.reportSubscriptions.find((ref) => ref.id === entityRef.id) + ) { + webhook.reportSubscriptions.push(entityRef); + } + + return webhook; + }), + switchMap((webhook) => + this.webhookRepository.storeWebhook(this.mapToEntity(webhook)), + ), + map(() => { + return null; + }), + ); + } + + removeSubscription( + webhookRef: Reference, + entityRef: Reference, + ): Observable { + return this.fetchWebhook(webhookRef).pipe( + map((webhook): Webhook => { + if (!webhook) { + throw new NotFoundException(); + } + + webhook.reportSubscriptions = webhook.reportSubscriptions.filter( + (ref) => ref.id !== entityRef.id, + ); + + return webhook; + }), + switchMap((webhook) => + this.webhookRepository.storeWebhook(this.mapToEntity(webhook)), + ), + map(() => { + return null; + }), + ); + } -@Injectable() -export class WebhookStorageService { /** * Get all registered webhooks subscribe by the user authenticated with the given token * @param token */ - fetchAllWebhooks(token: string): Observable { - return of([]); + fetchAllWebhooks(token?: string): Observable { + return this.webhookRepository + .fetchAllWebhooks() + .pipe( + map((entities) => entities.map((entity) => this.mapFromEntity(entity))), + ); } - fetchWebhook(webhook: Reference): Observable { - return of(undefined); + fetchWebhook(webhookRef: Reference): Observable { + return this.webhookRepository + .fetchWebhook(webhookRef) + .pipe(map((entity) => this.mapFromEntity(entity))); } /** * Creates a new webhook with the given configuration, stores it and returns a reference to the new webhook. - * @param webhookConfig + * @param request */ - createWebhook(webhookConfig: WebhookConfiguration): Observable { - return of(new Reference('new-webhook-id')); + createWebhook(request: CreateWebhookRequest): Observable { + return this.webhookRepository.storeWebhook({ + id: `Webhook:${uuidv4()}`, + label: request.label, + target: request.target, + authentication: { + type: 'API_KEY', + apiKey: this.cryptoService.encrypt(request.authentication.apiKey), + }, + owner: { + type: 'USER', + id: 'todo-user-id-here', + }, + reportSubscriptions: [], + }); + } + + private mapFromEntity(entity: WebhookEntity): Webhook { + return { + id: entity.id, + label: entity.label, + authentication: { + type: entity.authentication.type, + apiKey: this.cryptoService.decrypt(entity.authentication.apiKey), + }, + owner: entity.owner, + target: entity.target, + reportSubscriptions: entity.reportSubscriptions, + }; + } + + private mapToEntity(entity: Webhook): WebhookEntity { + return { + id: entity.id, + label: entity.label, + authentication: { + type: entity.authentication.type, + apiKey: this.cryptoService.encrypt(entity.authentication.apiKey), + }, + owner: entity.owner, + target: entity.target, + reportSubscriptions: entity.reportSubscriptions, + }; } } diff --git a/src/report-changes/core/report-changes.service.ts b/src/report-changes/core/report-changes.service.ts index 11a10c7..da19de9 100644 --- a/src/report-changes/core/report-changes.service.ts +++ b/src/report-changes/core/report-changes.service.ts @@ -91,7 +91,6 @@ export class ReportChangesService { .subscribe((affectedReports: ReportDataChangeEvent[]) => { affectedReports.forEach((event) => { this.notificationService.triggerNotification(event); - console.log('Report change detected:', event); }); }); } diff --git a/src/report/di/couchdb-sqs-configuration.ts b/src/report/di/couchdb-sqs-configuration.ts index 92a7c66..e19af0d 100644 --- a/src/report/di/couchdb-sqs-configuration.ts +++ b/src/report/di/couchdb-sqs-configuration.ts @@ -2,11 +2,11 @@ import { CouchSqsClient, CouchSqsClientConfig, } from '../../couchdb/couch-sqs.client'; -import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; +import { HttpService } from '@nestjs/axios'; +import axios from 'axios'; export const CouchSqsClientFactory = ( - httpService: HttpService, configService: ConfigService, ): CouchSqsClient => { const CONFIG_BASE = 'COUCH_SQS_CLIENT_CONFIG_'; @@ -19,10 +19,12 @@ export const CouchSqsClientFactory = ( ), }; - httpService.axiosRef.defaults.baseURL = config.BASE_URL; - httpService.axiosRef.defaults.headers['Authorization'] = `Basic ${Buffer.from( + const axiosInstance = axios.create(); + + axiosInstance.defaults.baseURL = config.BASE_URL; + axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, ).toString('base64')}`; - return new CouchSqsClient(httpService); + return new CouchSqsClient(new HttpService(axiosInstance)); }; diff --git a/src/report/report.module.ts b/src/report/report.module.ts index 190f795..d9ec523 100644 --- a/src/report/report.module.ts +++ b/src/report/report.module.ts @@ -1,7 +1,7 @@ import { Module } from '@nestjs/common'; import { DefaultReportStorage } from './storage/report-storage.service'; import { ReportController } from './controller/report.controller'; -import { HttpModule, HttpService } from '@nestjs/axios'; +import { HttpModule } from '@nestjs/axios'; import { ReportRepository } from './repository/report-repository.service'; import { ReportCalculationRepository } from './repository/report-calculation-repository.service'; import { ReportCalculationController } from './controller/report-calculation.controller'; @@ -28,7 +28,7 @@ import { CouchDbClient } from '../couchdb/couch-db-client.service'; { provide: CouchSqsClient, useFactory: CouchSqsClientFactory, - inject: [HttpService, ConfigService], + inject: [ConfigService], }, CreateReportCalculationUseCase, ], diff --git a/src/report/repository/report-calculation-repository.service.ts b/src/report/repository/report-calculation-repository.service.ts index bc02554..fc0bbf1 100644 --- a/src/report/repository/report-calculation-repository.service.ts +++ b/src/report/repository/report-calculation-repository.service.ts @@ -59,32 +59,30 @@ export class ReportCalculationRepository { storeCalculation( reportCalculation: ReportCalculation, ): Observable { - return this.couchDbClient.putDatabaseDocument( - this.databaseUrl, - this.databaseName, - reportCalculation.id, - reportCalculation, - { + return this.couchDbClient.putDatabaseDocument({ + documentId: `${this.databaseUrl}/${this.databaseName}/${reportCalculation.id}`, + body: reportCalculation, + config: { headers: { Authorization: this.authHeaderValue, }, }, - ); + }); } fetchCalculations(): Observable { return this.couchDbClient.getDatabaseDocument( - this.databaseUrl, - this.databaseName, - '_all_docs', { - params: { - include_docs: true, - start_key: '"ReportCalculation"', - end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character - }, - headers: { - Authorization: this.authHeaderValue, + documentId: `${this.databaseUrl}/${this.databaseName}/_all_docs`, + config: { + params: { + include_docs: true, + start_key: '"ReportCalculation"', + end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character + }, + headers: { + Authorization: this.authHeaderValue, + }, }, }, ); @@ -94,16 +92,14 @@ export class ReportCalculationRepository { calculationRef: Reference, ): Observable { return this.couchDbClient - .getDatabaseDocument( - this.databaseUrl, - this.databaseName, - calculationRef.id, - { + .getDatabaseDocument({ + documentId: `${this.databaseUrl}/${this.databaseName}/${calculationRef.id}`, + config: { headers: { Authorization: this.authHeaderValue, }, }, - ) + }) .pipe( map((rawReportCalculation) => new ReportCalculation( @@ -126,9 +122,13 @@ export class ReportCalculationRepository { storeData(data: ReportData): Observable { return this.couchDbClient - .putDatabaseDocument(this.databaseUrl, this.databaseName, data.id, data, { - headers: { - Authorization: this.authHeaderValue, + .putDatabaseDocument({ + documentId: `${this.databaseUrl}/${this.databaseName}/${data.id}`, + body: data, + config: { + headers: { + Authorization: this.authHeaderValue, + }, }, }) .pipe( @@ -139,21 +139,19 @@ export class ReportCalculationRepository { } calculation.setOutcome({ - result_hash: data.asHash(), + result_hash: data.getDataHash(), }); return this.couchDbClient - .putDatabaseDocument( - this.databaseUrl, - this.databaseName, - calculation.id, - calculation, - { + .putDatabaseDocument({ + documentId: `${this.databaseUrl}/${this.databaseName}/${calculation.id}`, + body: calculation, + config: { headers: { Authorization: this.authHeaderValue, }, }, - ) + }) .pipe(map(() => data)); }), ); @@ -170,17 +168,17 @@ export class ReportCalculationRepository { }), switchMap((calculationId) => { this.couchDbClient.getDatabaseDocument( - this.databaseUrl, - this.databaseName, - '_all_docs', { - params: { - include_docs: true, - start_key: '"' + calculationId + '"', - end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character - }, - headers: { - Authorization: this.authHeaderValue, + documentId: `${this.databaseUrl}/${this.databaseName}/_all_docs`, + config: { + params: { + include_docs: true, + start_key: '"' + calculationId + '"', + end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character + }, + headers: { + Authorization: this.authHeaderValue, + }, }, }, ); diff --git a/src/report/repository/report-repository.service.ts b/src/report/repository/report-repository.service.ts index beea45d..274fa55 100644 --- a/src/report/repository/report-repository.service.ts +++ b/src/report/repository/report-repository.service.ts @@ -1,4 +1,9 @@ -import { ForbiddenException, Injectable, NotFoundException, UnauthorizedException, } from '@nestjs/common'; +import { + ForbiddenException, + Injectable, + NotFoundException, + UnauthorizedException, +} from '@nestjs/common'; import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; import { catchError, map, Observable } from 'rxjs'; diff --git a/src/report/tasks/report-calculation-processor.service.ts b/src/report/tasks/report-calculation-processor.service.ts index 9c1a3d7..64d1294 100644 --- a/src/report/tasks/report-calculation-processor.service.ts +++ b/src/report/tasks/report-calculation-processor.service.ts @@ -74,7 +74,7 @@ export class ReportCalculationProcessor { reportCalculation .setStatus(ReportCalculationStatus.FINISHED_SUCCESS) .setOutcome({ - result_hash: reportData.asHash(), + result_hash: reportData.getDataHash(), }) .setEndDate(new Date().toISOString()), ); From 2f20d445aa188a4180e559e78ebeb79a994f1049 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 19 Feb 2024 07:33:48 +0100 Subject: [PATCH 11/18] feat: add url placeholder support for reportId --- .../core/notification.service.spec.ts | 32 ++++++++++++- src/notification/core/notification.service.ts | 19 +++++--- .../core/url-parser.service.spec.ts | 47 +++++++++++++++++++ src/notification/core/url-parser.service.ts | 25 ++++++++++ 4 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 src/notification/core/url-parser.service.spec.ts create mode 100644 src/notification/core/url-parser.service.ts diff --git a/src/notification/core/notification.service.spec.ts b/src/notification/core/notification.service.spec.ts index 65bd59d..b9e7463 100644 --- a/src/notification/core/notification.service.spec.ts +++ b/src/notification/core/notification.service.spec.ts @@ -1,12 +1,42 @@ import { Test, TestingModule } from '@nestjs/testing'; import { NotificationService } from './notification.service'; +import { WebhookStorage } from '../storage/webhook-storage.service'; +import { HttpModule, HttpService } from '@nestjs/axios'; +import { UrlParser } from './url-parser.service'; describe('NotificationService', () => { let service: NotificationService; + let mockWebhookStorage: { + fetchAllWebhooks: jest.Mock; + addSubscription: jest.Mock; + removeSubscription: jest.Mock; + }; + + let mockHttp: { request: jest.Mock }; + beforeEach(async () => { + mockWebhookStorage = { + fetchAllWebhooks: jest.fn(), + addSubscription: jest.fn(), + removeSubscription: jest.fn(), + }; + + mockHttp = { + request: jest.fn(), + }; + const module: TestingModule = await Test.createTestingModule({ - providers: [NotificationService], + imports: [HttpModule], + providers: [ + NotificationService, + { provide: HttpService, useValue: mockHttp }, + { + provide: WebhookStorage, + useValue: mockWebhookStorage, + }, + { provide: UrlParser, useClass: UrlParser }, + ], }).compile(); service = module.get(NotificationService); diff --git a/src/notification/core/notification.service.ts b/src/notification/core/notification.service.ts index 6adad2f..bc9b49f 100644 --- a/src/notification/core/notification.service.ts +++ b/src/notification/core/notification.service.ts @@ -13,6 +13,7 @@ import { ReportDataChangeEvent } from '../../domain/report-data-change-event'; import { WebhookStorage } from '../storage/webhook-storage.service'; import { HttpService } from '@nestjs/axios'; import { Webhook } from '../domain/webhook'; +import { UrlParser } from './url-parser.service'; /** * Manage core subscriptions and delivering events to subscribers. @@ -26,6 +27,7 @@ export class NotificationService { constructor( private webhookStorage: WebhookStorage, private httpService: HttpService, + private urlParser: UrlParser, ) {} /** @@ -63,11 +65,16 @@ export class NotificationService { ), ), mergeMap((webhooks) => - webhooks.map((webhook) => - this.httpService + webhooks.map((webhook) => { + // todo: support more placeholder and better checks + const url = this.urlParser.replacePlaceholder(webhook.target.url, { + reportId: event.report.id, + }); + + return this.httpService .request({ method: webhook.target.method, - url: webhook.target.url, + url: url, headers: { 'X-API-KEY': webhook.authentication.apiKey, }, @@ -81,13 +88,13 @@ export class NotificationService { console.log('could not send notification to webhook', err); throw err; }), - ), - ), + ); + }), ), zipAll(), ) .subscribe({ - next: (value) => { + next: () => { console.log('webhook send'); }, error: (err) => { diff --git a/src/notification/core/url-parser.service.spec.ts b/src/notification/core/url-parser.service.spec.ts new file mode 100644 index 0000000..59804b1 --- /dev/null +++ b/src/notification/core/url-parser.service.spec.ts @@ -0,0 +1,47 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { UrlParser } from './url-parser.service'; + +describe('UrlParserService', () => { + let service: UrlParser; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [UrlParser], + }).compile(); + + service = module.get(UrlParser); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should replace a path argument', () => { + const url = 'https://aam.internal/foo/bar//doo'; + + const target = service.replacePlaceholder(url, { + reportId: '123', + }); + + expect(target).toEqual('https://aam.internal/foo/bar/123/doo'); + }); + + it('should replace multiple path arguments', () => { + const url = 'https://aam.internal//bar//doo'; + + const target = service.replacePlaceholder(url, { + reportId: '123', + apiVersion: 'v1', + }); + + expect(target).toEqual('https://aam.internal/v1/bar/123/doo'); + }); + + it('should return all placeholder from url', () => { + const url = 'https://aam.internal//bar//doo'; + + const target: string[] = service.getPlaceholder(url); + + expect(target).toStrictEqual(['', '']); + }); +}); diff --git a/src/notification/core/url-parser.service.ts b/src/notification/core/url-parser.service.ts new file mode 100644 index 0000000..e1d0c65 --- /dev/null +++ b/src/notification/core/url-parser.service.ts @@ -0,0 +1,25 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class UrlParser { + getPlaceholder(url: string): string[] { + const pattern = /<([^<>]+)>/g; + + return url.match(pattern) || []; + } + + replacePlaceholder( + url: string, + args: { + [key: string]: string; + }, + ): string { + const pathParamsKeys = Object.keys(args); + + for (let i = 0; i < pathParamsKeys.length; i++) { + url = url.replace(`<${pathParamsKeys[i]}>`, args[pathParamsKeys[i]]); + } + + return url; + } +} From 5180f7233f4daca4ca5ce030afa401ef9d08df97 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 19 Feb 2024 07:41:35 +0100 Subject: [PATCH 12/18] fix: add missing provider --- src/notification/notification.module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/notification/notification.module.ts b/src/notification/notification.module.ts index 83fc596..3e1a49c 100644 --- a/src/notification/notification.module.ts +++ b/src/notification/notification.module.ts @@ -7,12 +7,14 @@ import { WebhookStorageFactory } from './di/notification-configuration'; import { CryptoModule } from '../crypto/crypto/crypto.module'; import { CryptoService } from '../crypto/core/crypto.service'; import { HttpModule } from '@nestjs/axios'; +import { UrlParser } from './core/url-parser.service'; @Module({ controllers: [WebhookController], imports: [CryptoModule, HttpModule], providers: [ NotificationService, + UrlParser, { provide: WebhookStorage, useFactory: WebhookStorageFactory, From 5725bc7bbdaa984320fc3461a83bbe567f6b0b26 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 19 Feb 2024 07:45:27 +0100 Subject: [PATCH 13/18] fix: return mapping --- .../repository/webhook-repository.service.ts | 6 ++-- .../storage/webhook-storage.service.ts | 30 ++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/notification/repository/webhook-repository.service.ts b/src/notification/repository/webhook-repository.service.ts index fbc3a7b..10669a6 100644 --- a/src/notification/repository/webhook-repository.service.ts +++ b/src/notification/repository/webhook-repository.service.ts @@ -2,7 +2,7 @@ import { NotFoundException } from '@nestjs/common'; import { catchError, map, Observable } from 'rxjs'; import { Reference } from '../../domain/reference'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; -import { CouchDbRow, CouchDbRows } from '../../couchdb/dtos'; +import { CouchDbRow, CouchDbRows, DocSuccess } from '../../couchdb/dtos'; export interface WebhookEntity { id: string; @@ -61,8 +61,8 @@ export class WebhookRepository { ); } - storeWebhook(webhook: WebhookEntity): Observable { - return this.http.putDatabaseDocument({ + storeWebhook(webhook: WebhookEntity): Observable { + return this.http.putDatabaseDocument({ documentId: webhook.id, body: webhook, config: {}, diff --git a/src/notification/storage/webhook-storage.service.ts b/src/notification/storage/webhook-storage.service.ts index 0eaf21a..b6c5236 100644 --- a/src/notification/storage/webhook-storage.service.ts +++ b/src/notification/storage/webhook-storage.service.ts @@ -90,20 +90,22 @@ export class WebhookStorage { * @param request */ createWebhook(request: CreateWebhookRequest): Observable { - return this.webhookRepository.storeWebhook({ - id: `Webhook:${uuidv4()}`, - label: request.label, - target: request.target, - authentication: { - type: 'API_KEY', - apiKey: this.cryptoService.encrypt(request.authentication.apiKey), - }, - owner: { - type: 'USER', - id: 'todo-user-id-here', - }, - reportSubscriptions: [], - }); + return this.webhookRepository + .storeWebhook({ + id: `Webhook:${uuidv4()}`, + label: request.label, + target: request.target, + authentication: { + type: 'API_KEY', + apiKey: this.cryptoService.encrypt(request.authentication.apiKey), + }, + owner: { + type: 'USER', + id: 'todo-user-id-here', + }, + reportSubscriptions: [], + }) + .pipe(map((value) => new Reference(value.id))); } private mapFromEntity(entity: WebhookEntity): Webhook { From 94e1cd9944afd62071f1c86aaef37b1c903efe16 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 19 Feb 2024 08:13:18 +0100 Subject: [PATCH 14/18] fix: small code fixes --- src/notification/controller/dtos.ts | 12 ++++++------ src/notification/controller/webhook.controller.ts | 1 + src/notification/storage/webhook-storage.service.ts | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/notification/controller/dtos.ts b/src/notification/controller/dtos.ts index 4292c6f..11af76a 100644 --- a/src/notification/controller/dtos.ts +++ b/src/notification/controller/dtos.ts @@ -1,3 +1,5 @@ +import { Reference } from '../../domain/reference'; + export interface WebhookDto { id: string; name: string; @@ -6,6 +8,7 @@ export interface WebhookDto { url: string; }; authenticationType: 'API_KEY'; + reportSubscriptions: Reference[]; } export interface CreateWebhookDto { @@ -14,13 +17,10 @@ export interface CreateWebhookDto { method: 'GET' | 'POST'; url: string; }; - authentication: { - type: 'API_KEY'; - apiKey: string; - }; + authentication: ApiKeyAuthConfig; } export interface ApiKeyAuthConfig { - key: string; - headerName: string; + type: 'API_KEY'; + apiKey: string; } diff --git a/src/notification/controller/webhook.controller.ts b/src/notification/controller/webhook.controller.ts index 8b364ed..5b12420 100644 --- a/src/notification/controller/webhook.controller.ts +++ b/src/notification/controller/webhook.controller.ts @@ -101,6 +101,7 @@ export class WebhookController { name: webhook.label, target: webhook.target, authenticationType: webhook.authentication.type, + reportSubscriptions: webhook.reportSubscriptions, }; } } diff --git a/src/notification/storage/webhook-storage.service.ts b/src/notification/storage/webhook-storage.service.ts index b6c5236..160e9c9 100644 --- a/src/notification/storage/webhook-storage.service.ts +++ b/src/notification/storage/webhook-storage.service.ts @@ -101,7 +101,7 @@ export class WebhookStorage { }, owner: { type: 'USER', - id: 'todo-user-id-here', + id: 'todo-user-id-here', // todo }, reportSubscriptions: [], }) From 9d1283b7b4edb78dd7feb300b863b4f165426c27 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 19 Feb 2024 08:34:03 +0100 Subject: [PATCH 15/18] fix: internal api change --- .../storage/couchdb-changes.service.ts | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/report-changes/storage/couchdb-changes.service.ts b/src/report-changes/storage/couchdb-changes.service.ts index d8ec28e..b68db0d 100644 --- a/src/report-changes/storage/couchdb-changes.service.ts +++ b/src/report-changes/storage/couchdb-changes.service.ts @@ -25,7 +25,12 @@ import { } from 'rxjs'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; import { CouchDbChangesResponse } from '../../couchdb/dtos'; -import { DatabaseChangeResult, DatabaseChangesService, DocChangeDetails, EntityDoc, } from './database-changes.service'; +import { + DatabaseChangeResult, + DatabaseChangesService, + DocChangeDetails, + EntityDoc, +} from './database-changes.service'; /** * Access _changes from a CouchDB @@ -39,11 +44,11 @@ export class CouchdbChangesService extends DatabaseChangesService { private databasePassword: string = this.configService.getOrThrow('DATABASE_PASSWORD'); - private changesPollInterval: number = Number( + private changesPollInterval = Number( this.configService.getOrThrow('CHANGES_POLL_INTERVAL'), ); - private authHeaderValue: string; + private readonly authHeaderValue: string; constructor( private couchdbClient: CouchDbClient, @@ -60,7 +65,7 @@ export class CouchdbChangesService extends DatabaseChangesService { private changesSubscription: Subscription | undefined; subscribeToAllNewChanges( - includeDocs: boolean = false, + includeDocs = false, ): Observable { if (!this.changesSubscription) { let lastSeq = 'now'; @@ -117,15 +122,13 @@ export class CouchdbChangesService extends DatabaseChangesService { return of(undefined); } - return this.couchdbClient.getDatabaseDocument( - this.dbUrl, - this.databaseName, - docId, - { + return this.couchdbClient.getDatabaseDocument({ + documentId: docId, + config: { params: { rev: previousRev }, headers: { Authorization: this.authHeaderValue }, }, - ); + }); }), ); } @@ -137,15 +140,13 @@ export class CouchdbChangesService extends DatabaseChangesService { */ private findLastRev(docId: string): Observable { return this.couchdbClient - .getDatabaseDocument( - this.dbUrl, - this.databaseName, - docId, - { + .getDatabaseDocument({ + documentId: docId, + config: { params: { revs_info: true }, headers: { Authorization: this.authHeaderValue }, }, - ) + }) .pipe( map((doc) => { const revsInfo: CouchDbDocRevsInfo = doc._revs_info; From 0c126d1a21f210e38558a2629dd8ade43edd3e0a Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 19 Feb 2024 14:26:21 +0100 Subject: [PATCH 16/18] feat: use custom di in all places --- .env | 20 ++--- build/Dockerfile | 8 -- src/config/app.yaml | 21 ++++- src/couchdb/couch-db-client.service.ts | 23 ++--- src/couchdb/couch-sqs.client.ts | 11 ++- src/couchdb/default-factory.ts | 53 +++++++++++ src/crypto/{crypto => }/crypto.module.ts | 4 +- src/notification/core/notification.service.ts | 45 +++++----- src/notification/core/url-parser.service.ts | 3 - .../di/notification-configuration.ts | 57 ++++++------ src/notification/notification.module.ts | 18 ++-- .../storage/webhook-storage.service.ts | 1 + .../core/report-changes.service.spec.ts | 8 +- .../core/report-changes.service.ts | 10 +-- .../di/report-changes-configuration.ts | 35 ++++++++ src/report-changes/report-changes.module.ts | 34 +++++-- ...ec.ts => couch-db-changes.service.spec.ts} | 30 +++---- ...service.ts => couch-db-changes.service.ts} | 68 +++++--------- src/report-changes/test-controller.ts | 20 ----- .../report-calculation.controller.spec.ts | 4 +- .../report-calculation.controller.ts | 4 +- .../controller/report.controller.spec.ts | 4 +- src/report/controller/report.controller.ts | 4 +- ...{report-storage.ts => i-report-storage.ts} | 2 +- .../sqs-report-calculator.service.spec.ts | 4 +- .../core/sqs-report-calculator.service.ts | 8 +- ...eport-calculation-use-case.service.spec.ts | 4 +- ...ate-report-calculation-use-case.service.ts | 6 +- src/report/di/couchdb-sqs-configuration.ts | 30 ------- src/report/di/report-configuration.ts | 48 ++++++++++ src/report/report.module.ts | 37 +++++--- .../report-calculation-repository.service.ts | 88 ++++--------------- .../repository/report-repository.service.ts | 70 +++++++-------- ...c.ts => reporting-storage.service.spec.ts} | 10 +-- ...ervice.ts => reporting-storage.service.ts} | 7 +- ...port-calculation-processor.service.spec.ts | 4 +- .../report-calculation-processor.service.ts | 4 +- 37 files changed, 414 insertions(+), 393 deletions(-) create mode 100644 src/couchdb/default-factory.ts rename src/crypto/{crypto => }/crypto.module.ts (73%) create mode 100644 src/report-changes/di/report-changes-configuration.ts rename src/report-changes/storage/{couchdb-changes.service.spec.ts => couch-db-changes.service.spec.ts} (87%) rename src/report-changes/storage/{couchdb-changes.service.ts => couch-db-changes.service.ts} (69%) delete mode 100644 src/report-changes/test-controller.ts rename src/report/core/{report-storage.ts => i-report-storage.ts} (96%) delete mode 100644 src/report/di/couchdb-sqs-configuration.ts create mode 100644 src/report/di/report-configuration.ts rename src/report/storage/{report-storage.service.spec.ts => reporting-storage.service.spec.ts} (79%) rename src/report/storage/{report-storage.service.ts => reporting-storage.service.ts} (96%) diff --git a/.env b/.env index a1c93bc..805c158 100644 --- a/.env +++ b/.env @@ -1,18 +1,14 @@ -SENTRY_DSN= -PORT= -DATABASE_URL=http://127.0.0.1:5984 -DATABASE_USER=admin -DATABASE_PASSWORD=docker -QUERY_URL=http://127.0.0.1:4984 -SCHEMA_CONFIG_ID=_design/sqlite:config +REPORT_COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_USER=admin +REPORT_COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker -CHANGES_POLL_INTERVAL=60000 +REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_BASIC_AUTH_USER=admin +REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_BASIC_AUTH_PASSWORD=docker -REPORT_DATABASE_URL=http://127.0.0.1:5984 -REPORT_DATABASE_NAME=report-calculation +REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_CALCULATION_BASIC_AUTH_USER=admin +REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_CALCULATION_BASIC_AUTH_PASSWORD=docker -COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_USER=admin -COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker +REPORT_CHANGES_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_USER=admin +REPORT_CHANGES_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker NOTIFICATION_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_USER=admin NOTIFICATION_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker diff --git a/build/Dockerfile b/build/Dockerfile index 09579bc..777433b 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -32,13 +32,5 @@ RUN npm ci --no-progress --only=production COPY --from=builder /app/dist ./dist COPY --from=builder /app/.env ./ -# (optional) The sentry DSN in order to send the error messages to sentry -ENV SENTRY_DSN="" -ENV PORT="" -ENV DATABASE_URL="" -ENV DATABASE_ADMIN="admin" -ENV DATABASE_PASSWORD="" -ENV QUERY_URL="" - CMD ["node", "dist/main"] diff --git a/src/config/app.yaml b/src/config/app.yaml index e506b77..72ecae1 100644 --- a/src/config/app.yaml +++ b/src/config/app.yaml @@ -1,7 +1,22 @@ -COUCH_SQS_CLIENT_CONFIG: - BASE_URL: http://localhost:4984 - NOTIFICATION: COUCH_DB_CLIENT_CONFIG: BASE_URL: http://localhost:5984 TARGET_DATABASE: notification-webhook + +REPORT: + COUCH_DB_CLIENT_CONFIG: + REPORT: + BASE_URL: http://localhost:5984 + TARGET_DATABASE: app + REPORT_CALCULATION: + BASE_URL: http://localhost:5984 + TARGET_DATABASE: report-calculation + COUCH_SQS_CLIENT_CONFIG: + BASE_URL: http://localhost:4984 + SCHEMA_DESIGN_CONFIG: /app/_design/sqlite:config + +REPORT_CHANGES: + COUCH_DB_CLIENT_CONFIG: + BASE_URL: http://localhost:5984 + TARGET_DATABASE: app + POLL_INTERVAL: 10000 diff --git a/src/couchdb/couch-db-client.service.ts b/src/couchdb/couch-db-client.service.ts index f261e2e..0c4ff18 100644 --- a/src/couchdb/couch-db-client.service.ts +++ b/src/couchdb/couch-db-client.service.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Logger } from '@nestjs/common'; import { catchError, map, Observable, of, switchMap } from 'rxjs'; import { HttpService } from '@nestjs/axios'; import { AxiosHeaders } from 'axios'; @@ -11,22 +11,14 @@ export class CouchDbClientConfig { BASIC_AUTH_PASSWORD = ''; } -@Injectable() export class CouchDbClient { private readonly logger = new Logger(CouchDbClient.name); constructor(private httpService: HttpService) {} - changes( - databaseUrl: string, - databaseName: string, - config?: any, - ): Observable { + changes(request: { config?: any }): Observable { return this.httpService - .get( - `${databaseUrl}/${databaseName}/_changes`, - config, - ) + .get(`/_changes`, request.config) .pipe( map((response) => { return response.data; @@ -66,14 +58,9 @@ export class CouchDbClient { ); } - find( - databaseUrl: string, - databaseName: string, - body: any, - config?: any, - ): Observable { + find(request: { query: object; config: any }): Observable { return this.httpService - .post(`${databaseUrl}/${databaseName}/_find`, body, config) + .post(`_find`, request.query, request.config) .pipe( map((response) => { return response.data; diff --git a/src/couchdb/couch-sqs.client.ts b/src/couchdb/couch-sqs.client.ts index 4df0472..a2c2fb9 100644 --- a/src/couchdb/couch-sqs.client.ts +++ b/src/couchdb/couch-sqs.client.ts @@ -6,6 +6,7 @@ export class CouchSqsClientConfig { BASE_URL = ''; BASIC_AUTH_USER = ''; BASIC_AUTH_PASSWORD = ''; + SCHEMA_DESIGN_CONFIG = ''; } export interface QueryRequest { @@ -16,9 +17,15 @@ export interface QueryRequest { export class CouchSqsClient { private readonly logger: Logger = new Logger(CouchSqsClient.name); - constructor(private httpService: HttpService) {} + constructor( + private httpService: HttpService, + private config: CouchSqsClientConfig, + ) {} - executeQuery(path: string, query: QueryRequest): Observable { + executeQuery( + query: QueryRequest, + path: string = this.config.SCHEMA_DESIGN_CONFIG, + ): Observable { return this.httpService.post(path, query).pipe( map((response) => response.data), catchError((err) => { diff --git a/src/couchdb/default-factory.ts b/src/couchdb/default-factory.ts new file mode 100644 index 0000000..87e3edf --- /dev/null +++ b/src/couchdb/default-factory.ts @@ -0,0 +1,53 @@ +import { ConfigService } from '@nestjs/config'; +import { CouchDbClient, CouchDbClientConfig } from './couch-db-client.service'; +import { HttpService } from '@nestjs/axios'; +import axios from 'axios'; +import { CouchSqsClient, CouchSqsClientConfig } from './couch-sqs.client'; + +export const DefaultCouchDbClientFactory = ( + configPrefix: string, + configService: ConfigService, +): CouchDbClient => { + const config: CouchDbClientConfig = { + BASE_URL: configService.getOrThrow(configPrefix + 'BASE_URL'), + TARGET_DATABASE: configService.getOrThrow(configPrefix + 'TARGET_DATABASE'), + BASIC_AUTH_USER: configService.getOrThrow(configPrefix + 'BASIC_AUTH_USER'), + BASIC_AUTH_PASSWORD: configService.getOrThrow( + configPrefix + 'BASIC_AUTH_PASSWORD', + ), + }; + + const axiosInstance = axios.create(); + + axiosInstance.defaults.baseURL = `${config.BASE_URL}/${config.TARGET_DATABASE}`; + axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( + `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, + ).toString('base64')}`; + + return new CouchDbClient(new HttpService(axiosInstance)); +}; + +export const DefaultCouchSqsClientFactory = ( + configPrefix: string, + configService: ConfigService, +): CouchSqsClient => { + const config: CouchSqsClientConfig = { + BASE_URL: configService.getOrThrow(configPrefix + 'BASE_URL'), + BASIC_AUTH_USER: configService.getOrThrow(configPrefix + 'BASIC_AUTH_USER'), + BASIC_AUTH_PASSWORD: configService.getOrThrow( + configPrefix + 'BASIC_AUTH_PASSWORD', + ), + SCHEMA_DESIGN_CONFIG: configService.getOrThrow( + configPrefix + 'SCHEMA_DESIGN_CONFIG', + ), + }; + + const axiosInstance = axios.create(); + + axiosInstance.defaults.baseURL = config.BASE_URL; + axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( + `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, + ).toString('base64')}`; + + return new CouchSqsClient(new HttpService(axiosInstance), config); +}; diff --git a/src/crypto/crypto/crypto.module.ts b/src/crypto/crypto.module.ts similarity index 73% rename from src/crypto/crypto/crypto.module.ts rename to src/crypto/crypto.module.ts index b0adf85..1b99c8d 100644 --- a/src/crypto/crypto/crypto.module.ts +++ b/src/crypto/crypto.module.ts @@ -1,6 +1,6 @@ import { Module } from '@nestjs/common'; -import { CryptoService } from '../core/crypto.service'; -import { CryptoServiceFactory } from '../di/crypto-configuration'; +import { CryptoService } from './core/crypto.service'; +import { CryptoServiceFactory } from './di/crypto-configuration'; import { ConfigModule, ConfigService } from '@nestjs/config'; @Module({ diff --git a/src/notification/core/notification.service.ts b/src/notification/core/notification.service.ts index bc9b49f..e610e26 100644 --- a/src/notification/core/notification.service.ts +++ b/src/notification/core/notification.service.ts @@ -1,7 +1,5 @@ -import { Injectable } from '@nestjs/common'; import { BehaviorSubject, - catchError, map, mergeMap, Observable, @@ -18,7 +16,6 @@ import { UrlParser } from './url-parser.service'; /** * Manage core subscriptions and delivering events to subscribers. */ -@Injectable() export class NotificationService { private _activeReports: BehaviorSubject = new BehaviorSubject( [] as Reference[], @@ -71,34 +68,36 @@ export class NotificationService { reportId: event.report.id, }); - return this.httpService - .request({ - method: webhook.target.method, - url: url, - headers: { - 'X-API-KEY': webhook.authentication.apiKey, - }, - timeout: 5000, - }) - .pipe( - map((response) => { - console.log('axios response', response); - }), - catchError((err) => { - console.log('could not send notification to webhook', err); - throw err; - }), - ); + return this.httpService.request({ + method: webhook.target.method, + url: url, + data: { + calculation_id: event.calculation.id, + }, + headers: { + Authorization: `Token ${webhook.authentication.apiKey}`, + }, + timeout: 5000, + }); }), ), zipAll(), ) .subscribe({ next: () => { - console.log('webhook send'); + console.log('webhook called successfully'); }, error: (err) => { - console.log(err); + console.log('could not send notification to webhook', { + error: { + code: err.code, + response: { + status: err.response.status, + statusText: err.response.statusText, + data: err.response.data, + }, + }, + }); }, complete: () => { console.log('webhook trigger completed'); diff --git a/src/notification/core/url-parser.service.ts b/src/notification/core/url-parser.service.ts index e1d0c65..bf7cfdd 100644 --- a/src/notification/core/url-parser.service.ts +++ b/src/notification/core/url-parser.service.ts @@ -1,6 +1,3 @@ -import { Injectable } from '@nestjs/common'; - -@Injectable() export class UrlParser { getPlaceholder(url: string): string[] { const pattern = /<([^<>]+)>/g; diff --git a/src/notification/di/notification-configuration.ts b/src/notification/di/notification-configuration.ts index 85c1d8f..39c1fcd 100644 --- a/src/notification/di/notification-configuration.ts +++ b/src/notification/di/notification-configuration.ts @@ -1,43 +1,40 @@ -import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; -import { - CouchDbClient, - CouchDbClientConfig, -} from '../../couchdb/couch-db-client.service'; import { WebhookStorage } from '../storage/webhook-storage.service'; import { WebhookRepository } from '../repository/webhook-repository.service'; import { CryptoService } from '../../crypto/core/crypto.service'; +import { DefaultCouchDbClientFactory } from '../../couchdb/default-factory'; +import { NotificationService } from '../core/notification.service'; +import { HttpService } from '@nestjs/axios'; import axios from 'axios'; - -export const CouchDbClientFactory = ( - configService: ConfigService, -): CouchDbClient => { - const CONFIG_BASE = 'NOTIFICATION_COUCH_DB_CLIENT_CONFIG_'; - - const config: CouchDbClientConfig = { - BASE_URL: configService.getOrThrow(CONFIG_BASE + 'BASE_URL'), - TARGET_DATABASE: configService.getOrThrow(CONFIG_BASE + 'TARGET_DATABASE'), - BASIC_AUTH_USER: configService.getOrThrow(CONFIG_BASE + 'BASIC_AUTH_USER'), - BASIC_AUTH_PASSWORD: configService.getOrThrow( - CONFIG_BASE + 'BASIC_AUTH_PASSWORD', - ), - }; - - const axiosInstance = axios.create(); - - axiosInstance.defaults.baseURL = `${config.BASE_URL}/${config.TARGET_DATABASE}`; - axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( - `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, - ).toString('base64')}`; - - return new CouchDbClient(new HttpService(axiosInstance)); -}; +import { UrlParser } from '../core/url-parser.service'; export const WebhookStorageFactory = ( cryptoService: CryptoService, configService: ConfigService, ): WebhookStorage => { - const couchDbClient = CouchDbClientFactory(configService); + const couchDbClient = DefaultCouchDbClientFactory( + 'NOTIFICATION_COUCH_DB_CLIENT_CONFIG_', + configService, + ); const webhookRepository = new WebhookRepository(couchDbClient); return new WebhookStorage(webhookRepository, cryptoService); }; + +export const NotificationServiceFactory = ( + webhookStorage: WebhookStorage, +): NotificationService => { + return new NotificationService( + webhookStorage, + WebhookWebClient(), + new UrlParser(), + ); +}; + +export const WebhookWebClient = (): HttpService => { + const axiosInstance = axios.create(); + axiosInstance.interceptors.request.use((config) => { + console.log('Execute Webhook: ', config.url); + return config; + }); + return new HttpService(axiosInstance); +}; diff --git a/src/notification/notification.module.ts b/src/notification/notification.module.ts index 3e1a49c..cdb1d0a 100644 --- a/src/notification/notification.module.ts +++ b/src/notification/notification.module.ts @@ -3,23 +3,27 @@ import { NotificationService } from './core/notification.service'; import { WebhookStorage } from './storage/webhook-storage.service'; import { WebhookController } from './controller/webhook.controller'; import { ConfigService } from '@nestjs/config'; -import { WebhookStorageFactory } from './di/notification-configuration'; -import { CryptoModule } from '../crypto/crypto/crypto.module'; +import { + NotificationServiceFactory, + WebhookStorageFactory, +} from './di/notification-configuration'; +import { CryptoModule } from '../crypto/crypto.module'; import { CryptoService } from '../crypto/core/crypto.service'; -import { HttpModule } from '@nestjs/axios'; -import { UrlParser } from './core/url-parser.service'; @Module({ controllers: [WebhookController], - imports: [CryptoModule, HttpModule], + imports: [CryptoModule], providers: [ - NotificationService, - UrlParser, { provide: WebhookStorage, useFactory: WebhookStorageFactory, inject: [CryptoService, ConfigService], }, + { + provide: NotificationService, + useFactory: NotificationServiceFactory, + inject: [WebhookStorage], + }, ], exports: [NotificationService], }) diff --git a/src/notification/storage/webhook-storage.service.ts b/src/notification/storage/webhook-storage.service.ts index 160e9c9..5a372fc 100644 --- a/src/notification/storage/webhook-storage.service.ts +++ b/src/notification/storage/webhook-storage.service.ts @@ -9,6 +9,7 @@ import { v4 as uuidv4 } from 'uuid'; import { CryptoService } from '../../crypto/core/crypto.service'; import { NotFoundException } from '@nestjs/common'; +// todo interface export class WebhookStorage { constructor( private webhookRepository: WebhookRepository, diff --git a/src/report-changes/core/report-changes.service.spec.ts b/src/report-changes/core/report-changes.service.spec.ts index ba18622..bdc295b 100644 --- a/src/report-changes/core/report-changes.service.spec.ts +++ b/src/report-changes/core/report-changes.service.spec.ts @@ -3,8 +3,8 @@ import { ReportChangesService } from './report-changes.service'; import { BehaviorSubject, map, of, Subject } from 'rxjs'; import { NotificationService } from '../../notification/core/notification.service'; import { Reference } from '../../domain/reference'; -import { DefaultReportStorage } from '../../report/storage/report-storage.service'; -import { CouchdbChangesService } from '../storage/couchdb-changes.service'; +import { ReportingStorage } from '../../report/storage/reporting-storage.service'; +import { CouchDbChangesService } from '../storage/couch-db-changes.service'; import { CreateReportCalculationUseCase } from '../../report/core/use-cases/create-report-calculation-use-case.service'; import { DatabaseChangeResult } from '../storage/database-changes.service'; @@ -31,11 +31,11 @@ describe('ReportChangesService', () => { ReportChangesService, { provide: NotificationService, useValue: mockNotificationService }, { - provide: DefaultReportStorage, + provide: ReportingStorage, useValue: { fetchReport: () => of() }, }, { - provide: CouchdbChangesService, + provide: CouchDbChangesService, useValue: { subscribeToAllNewChanges: () => mockedChangesStream }, }, { diff --git a/src/report-changes/core/report-changes.service.ts b/src/report-changes/core/report-changes.service.ts index 70c95dc..3714ca7 100644 --- a/src/report-changes/core/report-changes.service.ts +++ b/src/report-changes/core/report-changes.service.ts @@ -1,12 +1,11 @@ -import { Injectable } from '@nestjs/common'; import { ReportChangeDetector } from './report-change-detector'; import { NotificationService } from '../../notification/core/notification.service'; import { Reference } from '../../domain/reference'; import { ReportDataChangeEvent } from '../../domain/report-data-change-event'; import { ReportCalculationOutcomeSuccess } from '../../domain/report-calculation'; import { Report } from '../../domain/report'; -import { CouchdbChangesService } from '../storage/couchdb-changes.service'; -import { DefaultReportStorage } from '../../report/storage/report-storage.service'; +import { CouchDbChangesService } from '../storage/couch-db-changes.service'; +import { ReportingStorage } from '../../report/storage/reporting-storage.service'; import { filter, map, Observable, switchMap, tap, zip } from 'rxjs'; import { CreateReportCalculationFailed, @@ -17,14 +16,13 @@ import { DocChangeDetails, } from '../storage/database-changes.service'; -@Injectable() export class ReportChangesService { private reportMonitors = new Map(); constructor( private notificationService: NotificationService, - private reportStorage: DefaultReportStorage, - private couchdbChangesRepository: CouchdbChangesService, + private reportStorage: ReportingStorage, + private couchdbChangesRepository: CouchDbChangesService, private createReportCalculation: CreateReportCalculationUseCase, ) { this.notificationService diff --git a/src/report-changes/di/report-changes-configuration.ts b/src/report-changes/di/report-changes-configuration.ts new file mode 100644 index 0000000..fbb7413 --- /dev/null +++ b/src/report-changes/di/report-changes-configuration.ts @@ -0,0 +1,35 @@ +import { CouchDbChangesService } from '../storage/couch-db-changes.service'; +import { DefaultCouchDbClientFactory } from '../../couchdb/default-factory'; +import { ConfigService } from '@nestjs/config'; +import { ReportChangesService } from '../core/report-changes.service'; +import { NotificationService } from '../../notification/core/notification.service'; +import { CreateReportCalculationUseCase } from '../../report/core/use-cases/create-report-calculation-use-case.service'; +import { ReportingStorage } from '../../report/storage/reporting-storage.service'; + +export const CouchdbChangesServiceFactory = ( + configService: ConfigService, +): CouchDbChangesService => { + return new CouchDbChangesService( + DefaultCouchDbClientFactory( + 'REPORT_CHANGES_COUCH_DB_CLIENT_CONFIG_', + configService, + ), + { + POLL_INTERVAL: configService.getOrThrow('REPORT_CHANGES_POLL_INTERVAL'), + }, + ); +}; + +export const ReportChangesServiceFactory = ( + notificationService: NotificationService, + reportStorage: ReportingStorage, + couchdbChangesService: CouchDbChangesService, + createReportCalculationUseCase: CreateReportCalculationUseCase, +): ReportChangesService => { + return new ReportChangesService( + notificationService, + reportStorage, + couchdbChangesService, + createReportCalculationUseCase, + ); +}; diff --git a/src/report-changes/report-changes.module.ts b/src/report-changes/report-changes.module.ts index cb8b0f9..1b85152 100644 --- a/src/report-changes/report-changes.module.ts +++ b/src/report-changes/report-changes.module.ts @@ -1,19 +1,35 @@ import { Module } from '@nestjs/common'; import { ReportChangesService } from './core/report-changes.service'; -import { CouchdbChangesService } from './storage/couchdb-changes.service'; +import { CouchDbChangesService } from './storage/couch-db-changes.service'; import { NotificationModule } from '../notification/notification.module'; import { ReportModule } from '../report/report.module'; -import { CouchDbClient } from '../couchdb/couch-db-client.service'; -import { HttpModule } from '@nestjs/axios'; -import { TestController } from './test-controller'; +import { + CouchdbChangesServiceFactory, + ReportChangesServiceFactory, +} from './di/report-changes-configuration'; +import { ConfigService } from '@nestjs/config'; +import { NotificationService } from '../notification/core/notification.service'; +import { ReportingStorage } from '../report/storage/reporting-storage.service'; +import { CreateReportCalculationUseCase } from '../report/core/use-cases/create-report-calculation-use-case.service'; @Module({ - controllers: [TestController], - imports: [NotificationModule, ReportModule, HttpModule], + imports: [NotificationModule, ReportModule], providers: [ - ReportChangesService, - CouchdbChangesService, - CouchDbClient, // TODO: pack this into a CouchDbModule together with HttpModule import etc. + { + provide: ReportChangesService, + useFactory: ReportChangesServiceFactory, + inject: [ + NotificationService, + ReportingStorage, + CouchDbChangesService, + CreateReportCalculationUseCase, + ], + }, + { + provide: CouchDbChangesService, + useFactory: CouchdbChangesServiceFactory, + inject: [ConfigService], + }, ], exports: [ReportChangesService], }) diff --git a/src/report-changes/storage/couchdb-changes.service.spec.ts b/src/report-changes/storage/couch-db-changes.service.spec.ts similarity index 87% rename from src/report-changes/storage/couchdb-changes.service.spec.ts rename to src/report-changes/storage/couch-db-changes.service.spec.ts index 945ca46..49977e1 100644 --- a/src/report-changes/storage/couchdb-changes.service.spec.ts +++ b/src/report-changes/storage/couch-db-changes.service.spec.ts @@ -1,14 +1,12 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { CouchdbChangesService } from './couchdb-changes.service'; -import { HttpModule } from '@nestjs/axios'; -import { ConfigService } from '@nestjs/config'; -import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { CouchDbChangesService } from './couch-db-changes.service'; import { finalize, of } from 'rxjs'; import { CouchDbChangesResponse } from '../../couchdb/dtos'; import { DatabaseChangeResult } from './database-changes.service'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; -describe('CouchdbChangesService', () => { - let service: CouchdbChangesService; +describe('CouchDbChangesService', () => { + let service: CouchDbChangesService; let mockCouchdbChanges: jest.Mock; @@ -21,6 +19,7 @@ describe('CouchdbChangesService', () => { beforeEach(async () => { changesRequestCounter = 0; + mockCouchdbChanges = jest.fn().mockImplementation(() => { changesRequestCounter++; return of({ @@ -31,27 +30,26 @@ describe('CouchdbChangesService', () => { }); const module: TestingModule = await Test.createTestingModule({ - imports: [HttpModule], + imports: [], providers: [ - CouchdbChangesService, { provide: CouchDbClient, useValue: { changes: mockCouchdbChanges } }, { - provide: ConfigService, - useValue: { - getOrThrow: jest.fn(() => { - return 'foo'; - }), + provide: CouchDbChangesService, + useFactory: (couchdbClient) => { + return new CouchDbChangesService(couchdbClient, { + POLL_INTERVAL: '10000', + }); }, + inject: [CouchDbClient], }, ], }).compile(); - service = module.get(CouchdbChangesService); - }); + service = module.get(CouchDbChangesService); - beforeEach(() => { jest.useFakeTimers(); }); + afterEach(() => { jest.runOnlyPendingTimers(); jest.useRealTimers(); diff --git a/src/report-changes/storage/couchdb-changes.service.ts b/src/report-changes/storage/couch-db-changes.service.ts similarity index 69% rename from src/report-changes/storage/couchdb-changes.service.ts rename to src/report-changes/storage/couch-db-changes.service.ts index b68db0d..4d27494 100644 --- a/src/report-changes/storage/couchdb-changes.service.ts +++ b/src/report-changes/storage/couch-db-changes.service.ts @@ -1,11 +1,9 @@ import { ForbiddenException, - Injectable, InternalServerErrorException, NotFoundException, UnauthorizedException, } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { catchError, EMPTY, @@ -32,42 +30,28 @@ import { EntityDoc, } from './database-changes.service'; +export class CouchDbChangesConfig { + POLL_INTERVAL = ''; +} + /** * Access _changes from a CouchDB */ -@Injectable() -export class CouchdbChangesService extends DatabaseChangesService { - // TODO: centralize this config by refactoring couchdbClient and providing configured clients through DI - private dbUrl: string = this.configService.getOrThrow('DATABASE_URL'); - private databaseName = 'app'; // TODO: move to config and clean up .env, clarifying different DBs there - private databaseUser: string = this.configService.getOrThrow('DATABASE_USER'); - private databasePassword: string = - this.configService.getOrThrow('DATABASE_PASSWORD'); - - private changesPollInterval = Number( - this.configService.getOrThrow('CHANGES_POLL_INTERVAL'), - ); - - private readonly authHeaderValue: string; - +export class CouchDbChangesService extends DatabaseChangesService { constructor( private couchdbClient: CouchDbClient, - private configService: ConfigService, + private config: CouchDbChangesConfig, ) { super(); - const authHeader = Buffer.from( - `${this.databaseUser}:${this.databasePassword}`, - ).toString('base64'); - this.authHeaderValue = `Basic ${authHeader}`; } - private changesSubj = new ReplaySubject(1); - private changesSubscription: Subscription | undefined; + private _changesSubj = new ReplaySubject(1); + private _changesSubscription: Subscription | undefined; subscribeToAllNewChanges( includeDocs = false, ): Observable { - if (!this.changesSubscription) { + if (!this._changesSubscription) { let lastSeq = 'now'; const changesFeed = of({}).pipe( mergeMap(() => this.fetchChanges(lastSeq, true, includeDocs)), @@ -75,22 +59,21 @@ export class CouchdbChangesService extends DatabaseChangesService { tap((res) => (lastSeq = res.last_seq)), // poll regularly to get latest changes repeat({ - delay: this.changesPollInterval, + delay: Number.parseInt(this.config.POLL_INTERVAL), }), - tap((res) => console.log('incoming couchdb changes', res)), ); - this.changesSubscription = changesFeed + this._changesSubscription = changesFeed .pipe(map((res) => res.results)) - .subscribe(this.changesSubj); + .subscribe(this._changesSubj); } - return this.changesSubj.asObservable().pipe( + return this._changesSubj.asObservable().pipe( finalize(() => { - if (!this.changesSubj.observed) { + if (!this._changesSubj.observed) { // stop polling - this.changesSubscription?.unsubscribe(); - this.changesSubscription = undefined; + this._changesSubscription?.unsubscribe(); + this._changesSubscription = undefined; } }), ); @@ -99,7 +82,7 @@ export class CouchdbChangesService extends DatabaseChangesService { subscribeToAllNewChangesWithDocs(): Observable { return this.subscribeToAllNewChanges(true).pipe( mergeAll(), - tap((change) => console.log('new couchdb change', change)), + tap((change) => console.debug('new couchdb change', change)), switchMap((change) => this.getPreviousRevOfDoc(change.id).pipe( map((doc) => ({ @@ -109,7 +92,7 @@ export class CouchdbChangesService extends DatabaseChangesService { })), ), ), - tap((change) => console.log('new change details', change)), + tap((change) => console.debug('new change details', change)), ); } @@ -126,7 +109,6 @@ export class CouchdbChangesService extends DatabaseChangesService { documentId: docId, config: { params: { rev: previousRev }, - headers: { Authorization: this.authHeaderValue }, }, }); }), @@ -144,7 +126,6 @@ export class CouchdbChangesService extends DatabaseChangesService { documentId: docId, config: { params: { revs_info: true }, - headers: { Authorization: this.authHeaderValue }, }, }) .pipe( @@ -172,13 +153,12 @@ export class CouchdbChangesService extends DatabaseChangesService { includeDocs = false, ): Observable { return this.couchdbClient - .changes(this.dbUrl, this.databaseName, { - params: { - since: since, - include_docs: includeDocs, - }, - headers: { - Authorization: this.authHeaderValue, + .changes({ + config: { + params: { + since: since, + include_docs: includeDocs, + }, }, }) .pipe( diff --git a/src/report-changes/test-controller.ts b/src/report-changes/test-controller.ts deleted file mode 100644 index 25dd530..0000000 --- a/src/report-changes/test-controller.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Controller, Get } from '@nestjs/common'; -import { ReportChangesService } from './core/report-changes.service'; -import { Reference } from '../domain/reference'; -import { NotificationService } from '../notification/core/notification.service'; - -// TODO: remove as soon as webhooks are implemented! -@Controller('/test') -export class TestController { - constructor( - private changeDetectionService: ReportChangesService, - private notificationService: NotificationService, - ) {} - - @Get('/register') - register() { - return this.changeDetectionService - .registerReportMonitoring(new Reference('ReportConfig:1')) - .catch((e) => console.log(e)); - } -} diff --git a/src/report/controller/report-calculation.controller.spec.ts b/src/report/controller/report-calculation.controller.spec.ts index a9fefa1..458b494 100644 --- a/src/report/controller/report-calculation.controller.spec.ts +++ b/src/report/controller/report-calculation.controller.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ReportCalculationController } from './report-calculation.controller'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { HttpModule } from '@nestjs/axios'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; import { ReportController } from './report.controller'; @@ -18,7 +18,7 @@ describe('ReportCalculationController', () => { imports: [HttpModule], providers: [ CouchDbClient, - DefaultReportStorage, + ReportingStorage, ReportController, ReportCalculationRepository, ReportRepository, diff --git a/src/report/controller/report-calculation.controller.ts b/src/report/controller/report-calculation.controller.ts index 9b27412..3abe99d 100644 --- a/src/report/controller/report-calculation.controller.ts +++ b/src/report/controller/report-calculation.controller.ts @@ -7,7 +7,7 @@ import { Param, Post, } from '@nestjs/common'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { map, Observable, switchMap } from 'rxjs'; import { ReportCalculation } from '../../domain/report-calculation'; import { Reference } from '../../domain/reference'; @@ -20,7 +20,7 @@ import { @Controller('/api/v1/reporting') export class ReportCalculationController { constructor( - private reportStorage: DefaultReportStorage, + private reportStorage: ReportingStorage, private createReportCalculation: CreateReportCalculationUseCase, ) {} diff --git a/src/report/controller/report.controller.spec.ts b/src/report/controller/report.controller.spec.ts index bdb63be..851e1ac 100644 --- a/src/report/controller/report.controller.spec.ts +++ b/src/report/controller/report.controller.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ReportController } from './report.controller'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { ReportRepository } from '../repository/report-repository.service'; import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; import { HttpModule } from '@nestjs/axios'; @@ -15,7 +15,7 @@ describe('ReportController', () => { imports: [HttpModule], providers: [ CouchDbClient, - DefaultReportStorage, + ReportingStorage, ReportController, ReportRepository, ReportCalculationRepository, diff --git a/src/report/controller/report.controller.ts b/src/report/controller/report.controller.ts index 2f8a47d..4afbfd8 100644 --- a/src/report/controller/report.controller.ts +++ b/src/report/controller/report.controller.ts @@ -7,14 +7,14 @@ import { switchMap, zipAll, } from 'rxjs'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { ReportDto } from './dtos'; import { Reference } from '../../domain/reference'; import { Report } from '../../domain/report'; @Controller('/api/v1/reporting') export class ReportController { - constructor(private reportStorage: DefaultReportStorage) {} + constructor(private reportStorage: ReportingStorage) {} @Get('/report') fetchReports( diff --git a/src/report/core/report-storage.ts b/src/report/core/i-report-storage.ts similarity index 96% rename from src/report/core/report-storage.ts rename to src/report/core/i-report-storage.ts index a889c1c..1d3f950 100644 --- a/src/report/core/report-storage.ts +++ b/src/report/core/i-report-storage.ts @@ -4,7 +4,7 @@ import { Observable, Subject } from 'rxjs'; import { ReportCalculation } from '../../domain/report-calculation'; import { ReportData } from '../../domain/report-data'; -export interface ReportStorage { +export interface IReportStorage { fetchAllReports(authToken: string, mode: string): Observable; fetchReport( diff --git a/src/report/core/sqs-report-calculator.service.spec.ts b/src/report/core/sqs-report-calculator.service.spec.ts index 59147a6..e893de9 100644 --- a/src/report/core/sqs-report-calculator.service.spec.ts +++ b/src/report/core/sqs-report-calculator.service.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { SqsReportCalculator } from './sqs-report-calculator.service'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { CouchSqsClient } from '../../couchdb/couch-sqs.client'; describe('SqsReportCalculatorService', () => { @@ -14,7 +14,7 @@ describe('SqsReportCalculatorService', () => { providers: [ SqsReportCalculator, { provide: CouchSqsClient, useValue: mockCouchSqsClient }, - { provide: DefaultReportStorage, useValue: mockReportStorage }, + { provide: ReportingStorage, useValue: mockReportStorage }, ], }).compile(); diff --git a/src/report/core/sqs-report-calculator.service.ts b/src/report/core/sqs-report-calculator.service.ts index 896bf36..3ea2743 100644 --- a/src/report/core/sqs-report-calculator.service.ts +++ b/src/report/core/sqs-report-calculator.service.ts @@ -1,6 +1,5 @@ import { BadRequestException, - Injectable, InternalServerErrorException, NotFoundException, } from '@nestjs/common'; @@ -8,16 +7,15 @@ import { ReportCalculator } from './report-calculator'; import { ReportData } from '../../domain/report-data'; import { map, mergeAll, Observable, switchMap } from 'rxjs'; import { ReportCalculation } from '../../domain/report-calculation'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { CouchSqsClient } from '../../couchdb/couch-sqs.client'; import { v4 as uuidv4 } from 'uuid'; import { Reference } from '../../domain/reference'; -@Injectable() export class SqsReportCalculator implements ReportCalculator { constructor( private sqsClient: CouchSqsClient, - private reportStorage: DefaultReportStorage, + private reportStorage: ReportingStorage, ) {} calculate(reportCalculation: ReportCalculation): Observable { @@ -37,7 +35,7 @@ export class SqsReportCalculator implements ReportCalculator { return report.queries.flatMap((query) => { return this.sqsClient - .executeQuery('/app/_design/sqlite:config', { + .executeQuery({ query: query, args: [], // TODO pass args here }) diff --git a/src/report/core/use-cases/create-report-calculation-use-case.service.spec.ts b/src/report/core/use-cases/create-report-calculation-use-case.service.spec.ts index ce801f2..f2b145c 100644 --- a/src/report/core/use-cases/create-report-calculation-use-case.service.spec.ts +++ b/src/report/core/use-cases/create-report-calculation-use-case.service.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { CreateReportCalculationUseCase } from './create-report-calculation-use-case.service'; -import { DefaultReportStorage } from '../../storage/report-storage.service'; +import { ReportingStorage } from '../../storage/reporting-storage.service'; describe('CreateReportCalculationUseCaseService', () => { let service: CreateReportCalculationUseCase; @@ -9,7 +9,7 @@ describe('CreateReportCalculationUseCaseService', () => { const module: TestingModule = await Test.createTestingModule({ providers: [ CreateReportCalculationUseCase, - { provide: DefaultReportStorage, useValue: {} }, + { provide: ReportingStorage, useValue: {} }, ], }).compile(); diff --git a/src/report/core/use-cases/create-report-calculation-use-case.service.ts b/src/report/core/use-cases/create-report-calculation-use-case.service.ts index 63c9124..8fa9cff 100644 --- a/src/report/core/use-cases/create-report-calculation-use-case.service.ts +++ b/src/report/core/use-cases/create-report-calculation-use-case.service.ts @@ -1,4 +1,3 @@ -import { Injectable } from '@nestjs/common'; import { Reference } from '../../../domain/reference'; import { filter, map, merge, Observable, take } from 'rxjs'; import { @@ -7,11 +6,10 @@ import { } from '../../../domain/report-calculation'; import { v4 as uuidv4 } from 'uuid'; import { Report } from '../../../domain/report'; -import { DefaultReportStorage } from '../../storage/report-storage.service'; +import { ReportingStorage } from '../../storage/reporting-storage.service'; -@Injectable() export class CreateReportCalculationUseCase { - constructor(private reportStorage: DefaultReportStorage) {} + constructor(private reportStorage: ReportingStorage) {} startReportCalculation( report: Report, diff --git a/src/report/di/couchdb-sqs-configuration.ts b/src/report/di/couchdb-sqs-configuration.ts deleted file mode 100644 index e19af0d..0000000 --- a/src/report/di/couchdb-sqs-configuration.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { - CouchSqsClient, - CouchSqsClientConfig, -} from '../../couchdb/couch-sqs.client'; -import { ConfigService } from '@nestjs/config'; -import { HttpService } from '@nestjs/axios'; -import axios from 'axios'; - -export const CouchSqsClientFactory = ( - configService: ConfigService, -): CouchSqsClient => { - const CONFIG_BASE = 'COUCH_SQS_CLIENT_CONFIG_'; - - const config: CouchSqsClientConfig = { - BASE_URL: configService.getOrThrow(CONFIG_BASE + 'BASE_URL'), - BASIC_AUTH_USER: configService.getOrThrow(CONFIG_BASE + 'BASIC_AUTH_USER'), - BASIC_AUTH_PASSWORD: configService.getOrThrow( - CONFIG_BASE + 'BASIC_AUTH_PASSWORD', - ), - }; - - const axiosInstance = axios.create(); - - axiosInstance.defaults.baseURL = config.BASE_URL; - axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( - `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, - ).toString('base64')}`; - - return new CouchSqsClient(new HttpService(axiosInstance)); -}; diff --git a/src/report/di/report-configuration.ts b/src/report/di/report-configuration.ts new file mode 100644 index 0000000..947af88 --- /dev/null +++ b/src/report/di/report-configuration.ts @@ -0,0 +1,48 @@ +import { + DefaultCouchDbClientFactory, + DefaultCouchSqsClientFactory, +} from '../../couchdb/default-factory'; +import { ConfigService } from '@nestjs/config'; +import { CouchSqsClient } from '../../couchdb/couch-sqs.client'; +import { ReportingStorage } from '../storage/reporting-storage.service'; +import { ReportRepository } from '../repository/report-repository.service'; +import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; +import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; +import { CreateReportCalculationUseCase } from '../core/use-cases/create-report-calculation-use-case.service'; + +export const ReportCouchSqsClientFactory = ( + configService: ConfigService, +): CouchSqsClient => + DefaultCouchSqsClientFactory( + 'REPORT_COUCH_SQS_CLIENT_CONFIG_', + configService, + ); + +export const ReportingStorageFactory = ( + configService: ConfigService, +): ReportingStorage => + new ReportingStorage( + new ReportRepository( + DefaultCouchDbClientFactory( + 'REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_', + configService, + ), + ), + new ReportCalculationRepository( + DefaultCouchDbClientFactory( + 'REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_CALCULATION_', + configService, + ), + ), + ); + +export const SqsReportCalculatorFactory = ( + couchSqsClient: CouchSqsClient, + reportingStorage: ReportingStorage, +): SqsReportCalculator => + new SqsReportCalculator(couchSqsClient, reportingStorage); + +export const CreateReportCalculationUseCaseFactory = ( + reportingStorage: ReportingStorage, +): CreateReportCalculationUseCase => + new CreateReportCalculationUseCase(reportingStorage); diff --git a/src/report/report.module.ts b/src/report/report.module.ts index d9ec523..20aed51 100644 --- a/src/report/report.module.ts +++ b/src/report/report.module.ts @@ -1,37 +1,48 @@ import { Module } from '@nestjs/common'; -import { DefaultReportStorage } from './storage/report-storage.service'; +import { ReportingStorage } from './storage/reporting-storage.service'; import { ReportController } from './controller/report.controller'; import { HttpModule } from '@nestjs/axios'; -import { ReportRepository } from './repository/report-repository.service'; -import { ReportCalculationRepository } from './repository/report-calculation-repository.service'; import { ReportCalculationController } from './controller/report-calculation.controller'; import { ReportCalculationTask } from './tasks/report-calculation-task.service'; import { ReportCalculationProcessor } from './tasks/report-calculation-processor.service'; import { SqsReportCalculator } from './core/sqs-report-calculator.service'; import { CreateReportCalculationUseCase } from './core/use-cases/create-report-calculation-use-case.service'; import { CouchSqsClient } from '../couchdb/couch-sqs.client'; -import { CouchSqsClientFactory } from './di/couchdb-sqs-configuration'; import { ConfigService } from '@nestjs/config'; -import { CouchDbClient } from '../couchdb/couch-db-client.service'; +import { + CreateReportCalculationUseCaseFactory, + ReportCouchSqsClientFactory, + ReportingStorageFactory, + SqsReportCalculatorFactory, +} from './di/report-configuration'; @Module({ controllers: [ReportController, ReportCalculationController], imports: [HttpModule], providers: [ - DefaultReportStorage, - ReportRepository, - ReportCalculationRepository, ReportCalculationTask, ReportCalculationProcessor, - SqsReportCalculator, - CouchDbClient, { provide: CouchSqsClient, - useFactory: CouchSqsClientFactory, + useFactory: ReportCouchSqsClientFactory, inject: [ConfigService], }, - CreateReportCalculationUseCase, + { + provide: ReportingStorage, + useFactory: ReportingStorageFactory, + inject: [ConfigService], + }, + { + provide: SqsReportCalculator, + useFactory: SqsReportCalculatorFactory, + inject: [CouchSqsClient, ReportingStorage], + }, + { + provide: CreateReportCalculationUseCase, + useFactory: CreateReportCalculationUseCaseFactory, + inject: [ReportingStorage], + }, ], - exports: [DefaultReportStorage, CreateReportCalculationUseCase], + exports: [ReportingStorage, CreateReportCalculationUseCase], }) export class ReportModule {} diff --git a/src/report/repository/report-calculation-repository.service.ts b/src/report/repository/report-calculation-repository.service.ts index fc0bbf1..ca9ef07 100644 --- a/src/report/repository/report-calculation-repository.service.ts +++ b/src/report/repository/report-calculation-repository.service.ts @@ -1,6 +1,5 @@ import { ForbiddenException, - Injectable, NotFoundException, UnauthorizedException, } from '@nestjs/common'; @@ -8,7 +7,6 @@ import { ReportCalculation } from '../../domain/report-calculation'; import { Reference } from '../../domain/reference'; import { ReportData } from '../../domain/report-data'; import { catchError, map, Observable, switchMap } from 'rxjs'; -import { ConfigService } from '@nestjs/config'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; import { DocSuccess, FindResponse } from '../../couchdb/dtos'; @@ -27,62 +25,29 @@ export interface FetchReportCalculationsResponse { rows: ReportCalculationEntity[]; } -@Injectable() export class ReportCalculationRepository { - static readonly REPORT_DATABASE_URL = 'REPORT_DATABASE_URL'; - static readonly REPORT_DATABASE_NAME = 'REPORT_DATABASE_NAME'; - - readonly databaseUrl: string; - readonly databaseName: string; - - readonly authHeaderValue: string; - - constructor( - private couchDbClient: CouchDbClient, - private configService: ConfigService, - ) { - this.databaseUrl = this.configService.getOrThrow( - ReportCalculationRepository.REPORT_DATABASE_URL, - ); - this.databaseName = this.configService.getOrThrow( - ReportCalculationRepository.REPORT_DATABASE_NAME, - ); - - const authHeader = Buffer.from( - `${this.configService.getOrThrow( - 'DATABASE_USER', - )}:${this.configService.getOrThrow('DATABASE_PASSWORD')}`, - ).toString('base64'); - this.authHeaderValue = `Basic ${authHeader}`; - } + constructor(private couchDbClient: CouchDbClient) {} storeCalculation( reportCalculation: ReportCalculation, ): Observable { return this.couchDbClient.putDatabaseDocument({ - documentId: `${this.databaseUrl}/${this.databaseName}/${reportCalculation.id}`, + documentId: reportCalculation.id, body: reportCalculation, - config: { - headers: { - Authorization: this.authHeaderValue, - }, - }, + config: {}, }); } fetchCalculations(): Observable { return this.couchDbClient.getDatabaseDocument( { - documentId: `${this.databaseUrl}/${this.databaseName}/_all_docs`, + documentId: `_all_docs`, config: { params: { include_docs: true, start_key: '"ReportCalculation"', end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character }, - headers: { - Authorization: this.authHeaderValue, - }, }, }, ); @@ -93,12 +58,8 @@ export class ReportCalculationRepository { ): Observable { return this.couchDbClient .getDatabaseDocument({ - documentId: `${this.databaseUrl}/${this.databaseName}/${calculationRef.id}`, - config: { - headers: { - Authorization: this.authHeaderValue, - }, - }, + documentId: `${calculationRef.id}`, + config: {}, }) .pipe( map((rawReportCalculation) => @@ -111,7 +72,7 @@ export class ReportCalculationRepository { .setEndDate(rawReportCalculation.end_date) .setOutcome(rawReportCalculation.outcome), ), - catchError((err, caught) => { + catchError((err) => { if (err.response.status === 404) { throw new NotFoundException(); } @@ -123,13 +84,9 @@ export class ReportCalculationRepository { storeData(data: ReportData): Observable { return this.couchDbClient .putDatabaseDocument({ - documentId: `${this.databaseUrl}/${this.databaseName}/${data.id}`, + documentId: `${data.id}`, body: data, - config: { - headers: { - Authorization: this.authHeaderValue, - }, - }, + config: {}, }) .pipe( switchMap(() => this.fetchCalculation(data.calculation)), @@ -144,13 +101,9 @@ export class ReportCalculationRepository { return this.couchDbClient .putDatabaseDocument({ - documentId: `${this.databaseUrl}/${this.databaseName}/${calculation.id}`, + documentId: `${calculation.id}`, body: calculation, - config: { - headers: { - Authorization: this.authHeaderValue, - }, - }, + config: {}, }) .pipe(map(() => data)); }), @@ -169,35 +122,26 @@ export class ReportCalculationRepository { switchMap((calculationId) => { this.couchDbClient.getDatabaseDocument( { - documentId: `${this.databaseUrl}/${this.databaseName}/_all_docs`, + documentId: `_all_docs`, config: { params: { include_docs: true, start_key: '"' + calculationId + '"', end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character }, - headers: { - Authorization: this.authHeaderValue, - }, }, }, ); return this.couchDbClient - .find>( - this.databaseUrl, - this.databaseName, - { + .find>({ + query: { selector: { 'calculation.id': { $eq: calculationId }, }, }, - { - headers: { - Authorization: this.authHeaderValue, - }, - }, - ) + config: {}, + }) .pipe( map((value) => { if (value.docs && value.docs.length === 1) { diff --git a/src/report/repository/report-repository.service.ts b/src/report/repository/report-repository.service.ts index 274fa55..221c66a 100644 --- a/src/report/repository/report-repository.service.ts +++ b/src/report/repository/report-repository.service.ts @@ -1,13 +1,11 @@ import { ForbiddenException, - Injectable, NotFoundException, UnauthorizedException, } from '@nestjs/common'; -import { HttpService } from '@nestjs/axios'; -import { ConfigService } from '@nestjs/config'; import { catchError, map, Observable } from 'rxjs'; import { CouchDbRow } from '../../couchdb/dtos'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; export interface ReportDoc { _id: string; @@ -31,38 +29,31 @@ interface FetchReportsResponse { rows: CouchDbRow[]; } -@Injectable() export class ReportRepository { - private dbUrl: string = this.configService.getOrThrow('DATABASE_URL'); - private databaseUser: string = this.configService.getOrThrow('DATABASE_USER'); - private databasePassword: string = - this.configService.getOrThrow('DATABASE_PASSWORD'); + constructor(private couchDbClient: CouchDbClient) {} - private authHeaderValue: string; + fetchReports(authToken?: string): Observable { + const config: any = { + params: { + include_docs: true, + startkey: '"ReportConfig:"', + endkey: '"ReportConfig:' + '\ufff0"', + }, + }; - constructor(private http: HttpService, private configService: ConfigService) { - const authHeader = Buffer.from( - `${this.databaseUser}:${this.databasePassword}`, - ).toString('base64'); - this.authHeaderValue = `Basic ${authHeader}`; - } + if (authToken) { + config.headers = { + Authorization: authToken, + }; + } - fetchReports( - authToken: string = this.authHeaderValue, - ): Observable { - return this.http - .get(`${this.dbUrl}/app/_all_docs`, { - params: { - include_docs: true, - startkey: '"ReportConfig:"', - endkey: '"ReportConfig:' + '\ufff0"', - }, - headers: { - Authorization: authToken, - }, + return this.couchDbClient + .getDatabaseDocument({ + documentId: `_all_docs`, + config: config, }) .pipe( - map((value) => value.data), + map((value) => value), catchError((err, caught) => { this.handleError(err); throw caught; @@ -72,16 +63,23 @@ export class ReportRepository { fetchReport( reportId: string, - authToken: string = this.authHeaderValue, + authToken?: string | undefined, ): Observable { - return this.http - .get(`${this.dbUrl}/app/${reportId}`, { - headers: { - Authorization: authToken, - }, + const config: any = {}; + + if (authToken) { + config.headers = { + Authorization: authToken, + }; + } + + return this.couchDbClient + .getDatabaseDocument({ + documentId: reportId, + config: config, }) .pipe( - map((value) => value.data), + map((value) => value), catchError((err, caught) => { this.handleError(err); throw caught; diff --git a/src/report/storage/report-storage.service.spec.ts b/src/report/storage/reporting-storage.service.spec.ts similarity index 79% rename from src/report/storage/report-storage.service.spec.ts rename to src/report/storage/reporting-storage.service.spec.ts index b88428d..63089f0 100644 --- a/src/report/storage/report-storage.service.spec.ts +++ b/src/report/storage/reporting-storage.service.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { ReportStorage } from '../core/report-storage'; -import { DefaultReportStorage } from './report-storage.service'; +import { IReportStorage } from '../core/i-report-storage'; +import { ReportingStorage } from './reporting-storage.service'; import { ReportRepository } from '../repository/report-repository.service'; import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; import { HttpModule } from '@nestjs/axios'; @@ -8,13 +8,13 @@ import { ConfigService } from '@nestjs/config'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; describe('DefaultReportStorage', () => { - let service: ReportStorage; + let service: IReportStorage; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ imports: [HttpModule], providers: [ - DefaultReportStorage, + ReportingStorage, ReportRepository, ReportCalculationRepository, CouchDbClient, @@ -29,7 +29,7 @@ describe('DefaultReportStorage', () => { ], }).compile(); - service = module.get(DefaultReportStorage); + service = module.get(ReportingStorage); }); it('should be defined', () => { diff --git a/src/report/storage/report-storage.service.ts b/src/report/storage/reporting-storage.service.ts similarity index 96% rename from src/report/storage/report-storage.service.ts rename to src/report/storage/reporting-storage.service.ts index 7bb4524..2758436 100644 --- a/src/report/storage/report-storage.service.ts +++ b/src/report/storage/reporting-storage.service.ts @@ -1,9 +1,9 @@ import { Reference } from '../../domain/reference'; import { Report } from '../../domain/report'; -import { ReportStorage } from '../core/report-storage'; +import { IReportStorage } from '../core/i-report-storage'; import { ReportRepository } from '../repository/report-repository.service'; import { map, Observable, Subject, switchMap, tap } from 'rxjs'; -import { Injectable, NotFoundException } from '@nestjs/common'; +import { NotFoundException } from '@nestjs/common'; import { ReportCalculation, ReportCalculationStatus, @@ -14,8 +14,7 @@ import { } from '../repository/report-calculation-repository.service'; import { ReportData } from '../../domain/report-data'; -@Injectable() -export class DefaultReportStorage implements ReportStorage { +export class ReportingStorage implements IReportStorage { constructor( private reportRepository: ReportRepository, private reportCalculationRepository: ReportCalculationRepository, diff --git a/src/report/tasks/report-calculation-processor.service.spec.ts b/src/report/tasks/report-calculation-processor.service.spec.ts index 413b87a..738a21f 100644 --- a/src/report/tasks/report-calculation-processor.service.spec.ts +++ b/src/report/tasks/report-calculation-processor.service.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ReportCalculationProcessor } from './report-calculation-processor.service'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; describe('ReportCalculationProcessorService', () => { @@ -14,7 +14,7 @@ describe('ReportCalculationProcessorService', () => { imports: [], providers: [ ReportCalculationProcessor, - { provide: DefaultReportStorage, useValue: mockReportStorage }, + { provide: ReportingStorage, useValue: mockReportStorage }, { provide: SqsReportCalculator, useValue: mockSqsReportCalculator }, ], }).compile(); diff --git a/src/report/tasks/report-calculation-processor.service.ts b/src/report/tasks/report-calculation-processor.service.ts index 64d1294..65d90b4 100644 --- a/src/report/tasks/report-calculation-processor.service.ts +++ b/src/report/tasks/report-calculation-processor.service.ts @@ -4,7 +4,7 @@ import { ReportCalculation, ReportCalculationStatus, } from '../../domain/report-calculation'; -import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportingStorage } from '../storage/reporting-storage.service'; import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; import { ReportData } from '../../domain/report-data'; @@ -13,7 +13,7 @@ export class ReportCalculationProcessor { private readonly logger = new Logger(ReportCalculationProcessor.name); constructor( - private reportStorage: DefaultReportStorage, + private reportStorage: ReportingStorage, private reportCalculator: SqsReportCalculator, ) {} From c6acbdf44a1f17c583dfc3e80acae394f043da53 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 19 Feb 2024 15:19:26 +0100 Subject: [PATCH 17/18] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f5378f..78a7c2e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Query Back'end +# Aam Digital - Query Backend +An API / microservice to calculate "reports" (e.g. statistical, summarized indicators) based on entities in the primary database of an Aam Digital instance. + This service allows to run SQL queries on the database. In particular, this service allows users with limited permissions to see reports of aggregated statistics across all data (e.g. a supervisor could analyse reports without having access to possibly confidential details of participants or notes). From d1b5108e7b945579b73d89391639482afbbb682d Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Mon, 19 Feb 2024 21:02:35 +0100 Subject: [PATCH 18/18] fix: pr feedback --- .env | 24 +++++----- README.md | 8 +--- src/config/app.yaml | 36 +++++++-------- src/couchdb/default-factory.ts | 26 ----------- .../di/notification-configuration.ts | 2 +- .../repository/webhook-repository.service.ts | 6 +++ .../di/report-changes-configuration.ts | 5 +- .../sqs-report-calculator.service.spec.ts | 2 +- .../core/sqs-report-calculator.service.ts | 2 +- src/report/di/report-configuration.ts | 46 +++++++++++++------ src/report/report.module.ts | 2 +- .../sqs}/couch-sqs-client.service.spec.ts | 0 .../sqs}/couch-sqs.client.ts | 0 13 files changed, 74 insertions(+), 85 deletions(-) rename src/{couchdb => report/sqs}/couch-sqs-client.service.spec.ts (100%) rename src/{couchdb => report/sqs}/couch-sqs.client.ts (100%) diff --git a/.env b/.env index 805c158..7553edf 100644 --- a/.env +++ b/.env @@ -1,16 +1,16 @@ -REPORT_COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_USER=admin -REPORT_COUCH_SQS_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker -REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_BASIC_AUTH_USER=admin -REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_BASIC_AUTH_PASSWORD=docker +; CouchDb Basic Auth credentials for different clients +COUCH_DB_CLIENT_NOTIFICATION_BASIC_AUTH_USER=admin +COUCH_DB_CLIENT_NOTIFICATION_BASIC_AUTH_PASSWORD=docker +COUCH_DB_CLIENT_REPORT_BASIC_AUTH_USER=admin +COUCH_DB_CLIENT_REPORT_BASIC_AUTH_PASSWORD=docker +COUCH_DB_CLIENT_REPORT_CALCULATION_BASIC_AUTH_USER=admin +COUCH_DB_CLIENT_REPORT_CALCULATION_BASIC_AUTH_PASSWORD=docker -REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_CALCULATION_BASIC_AUTH_USER=admin -REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_CALCULATION_BASIC_AUTH_PASSWORD=docker - -REPORT_CHANGES_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_USER=admin -REPORT_CHANGES_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker - -NOTIFICATION_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_USER=admin -NOTIFICATION_COUCH_DB_CLIENT_CONFIG_BASIC_AUTH_PASSWORD=docker +; SQS Basic Auth credentials +SQS_CLIENT_BASIC_AUTH_USER=admin +SQS_CLIENT_BASIC_AUTH_PASSWORD=docker +; Encryption Key for server side secret encryption +; e.g. webhook credentials stored encrypted in database CRYPTO_ENCRYPTION_SECRET=super-duper-secret diff --git a/README.md b/README.md index 78a7c2e..e0d9525 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,8 @@ See the [ndb-setup repo](https://github.com/Aam-Digital/ndb-setup) for full depl To use this you need a running [CouchDB](https://docs.couchdb.org/en/stable/) and [structured query server (SQS)](https://neighbourhood.ie/products-and-services/structured-query-server). -The following variables might need to be configured in the `.env` file: -- `DATABASE_URL` URL of the `CouchDB` or [replication backend](https://github.com/Aam-Digital/replication-backend) -- `QUERY_URL` URL of the SQS -- `SCHEMA_CONFIG_ID` database ID of the document which holds the SQS schema (default `_design/sqlite:config`) -- `PORT` where the app should listen (default 3000) -- `SENTRY_DSN` for remote logging +Config (e.g. database paths and credentials) can be provided as environment variables and/or through an `config/app.yaml` file in the root folder. +Secrets should be configured over `.env` files. Check out the example files. ----- # API access to reports diff --git a/src/config/app.yaml b/src/config/app.yaml index 72ecae1..83f5180 100644 --- a/src/config/app.yaml +++ b/src/config/app.yaml @@ -1,22 +1,22 @@ -NOTIFICATION: - COUCH_DB_CLIENT_CONFIG: +# CouchDb clients config +# BASE_URL: URL of the `CouchDB` or [replication backend](https://github.com/Aam-Digital/replication-backend) +# TARGET_DATABASE: CouchDb database name +COUCH_DB_CLIENT: + NOTIFICATION: BASE_URL: http://localhost:5984 TARGET_DATABASE: notification-webhook - -REPORT: - COUCH_DB_CLIENT_CONFIG: - REPORT: - BASE_URL: http://localhost:5984 - TARGET_DATABASE: app - REPORT_CALCULATION: - BASE_URL: http://localhost:5984 - TARGET_DATABASE: report-calculation - COUCH_SQS_CLIENT_CONFIG: - BASE_URL: http://localhost:4984 - SCHEMA_DESIGN_CONFIG: /app/_design/sqlite:config - -REPORT_CHANGES: - COUCH_DB_CLIENT_CONFIG: + REPORT: BASE_URL: http://localhost:5984 TARGET_DATABASE: app - POLL_INTERVAL: 10000 + REPORT_CALCULATION: + BASE_URL: http://localhost:5984 + TARGET_DATABASE: report-calculation + +# CouchDb SQS client config +# BASE_URL: URL of the SQS +# SCHEMA_DESIGN_CONFIG: database ID of the document which holds the SQS schema (default `/app/_design/sqlite:config`) +SQS_CLIENT: + BASE_URL: http://localhost:4984 + SCHEMA_DESIGN_CONFIG: /app/_design/sqlite:config + +REPORT_CHANGES_POLL_INTERVAL: 10000 diff --git a/src/couchdb/default-factory.ts b/src/couchdb/default-factory.ts index 87e3edf..03eea05 100644 --- a/src/couchdb/default-factory.ts +++ b/src/couchdb/default-factory.ts @@ -2,7 +2,6 @@ import { ConfigService } from '@nestjs/config'; import { CouchDbClient, CouchDbClientConfig } from './couch-db-client.service'; import { HttpService } from '@nestjs/axios'; import axios from 'axios'; -import { CouchSqsClient, CouchSqsClientConfig } from './couch-sqs.client'; export const DefaultCouchDbClientFactory = ( configPrefix: string, @@ -26,28 +25,3 @@ export const DefaultCouchDbClientFactory = ( return new CouchDbClient(new HttpService(axiosInstance)); }; - -export const DefaultCouchSqsClientFactory = ( - configPrefix: string, - configService: ConfigService, -): CouchSqsClient => { - const config: CouchSqsClientConfig = { - BASE_URL: configService.getOrThrow(configPrefix + 'BASE_URL'), - BASIC_AUTH_USER: configService.getOrThrow(configPrefix + 'BASIC_AUTH_USER'), - BASIC_AUTH_PASSWORD: configService.getOrThrow( - configPrefix + 'BASIC_AUTH_PASSWORD', - ), - SCHEMA_DESIGN_CONFIG: configService.getOrThrow( - configPrefix + 'SCHEMA_DESIGN_CONFIG', - ), - }; - - const axiosInstance = axios.create(); - - axiosInstance.defaults.baseURL = config.BASE_URL; - axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( - `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, - ).toString('base64')}`; - - return new CouchSqsClient(new HttpService(axiosInstance), config); -}; diff --git a/src/notification/di/notification-configuration.ts b/src/notification/di/notification-configuration.ts index 39c1fcd..35ce80c 100644 --- a/src/notification/di/notification-configuration.ts +++ b/src/notification/di/notification-configuration.ts @@ -13,7 +13,7 @@ export const WebhookStorageFactory = ( configService: ConfigService, ): WebhookStorage => { const couchDbClient = DefaultCouchDbClientFactory( - 'NOTIFICATION_COUCH_DB_CLIENT_CONFIG_', + 'COUCH_DB_CLIENT_NOTIFICATION_', configService, ); const webhookRepository = new WebhookRepository(couchDbClient); diff --git a/src/notification/repository/webhook-repository.service.ts b/src/notification/repository/webhook-repository.service.ts index 10669a6..e4d1e0d 100644 --- a/src/notification/repository/webhook-repository.service.ts +++ b/src/notification/repository/webhook-repository.service.ts @@ -4,6 +4,12 @@ import { Reference } from '../../domain/reference'; import { CouchDbClient } from '../../couchdb/couch-db-client.service'; import { CouchDbRow, CouchDbRows, DocSuccess } from '../../couchdb/dtos'; +/** + * Representation of a WebHook stored in the database. + * + * @field owner: represents the creator of the WebhookEntity. Normally the User/Client who created the Webhook. + * Just owners can read their own entities or the entities of groups they are part of. + */ export interface WebhookEntity { id: string; label: string; diff --git a/src/report-changes/di/report-changes-configuration.ts b/src/report-changes/di/report-changes-configuration.ts index fbb7413..4dcdbc1 100644 --- a/src/report-changes/di/report-changes-configuration.ts +++ b/src/report-changes/di/report-changes-configuration.ts @@ -10,10 +10,7 @@ export const CouchdbChangesServiceFactory = ( configService: ConfigService, ): CouchDbChangesService => { return new CouchDbChangesService( - DefaultCouchDbClientFactory( - 'REPORT_CHANGES_COUCH_DB_CLIENT_CONFIG_', - configService, - ), + DefaultCouchDbClientFactory('COUCH_DB_CLIENT_REPORT_', configService), { POLL_INTERVAL: configService.getOrThrow('REPORT_CHANGES_POLL_INTERVAL'), }, diff --git a/src/report/core/sqs-report-calculator.service.spec.ts b/src/report/core/sqs-report-calculator.service.spec.ts index e893de9..ddde523 100644 --- a/src/report/core/sqs-report-calculator.service.spec.ts +++ b/src/report/core/sqs-report-calculator.service.spec.ts @@ -1,7 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { SqsReportCalculator } from './sqs-report-calculator.service'; import { ReportingStorage } from '../storage/reporting-storage.service'; -import { CouchSqsClient } from '../../couchdb/couch-sqs.client'; +import { CouchSqsClient } from '../sqs/couch-sqs.client'; describe('SqsReportCalculatorService', () => { let service: SqsReportCalculator; diff --git a/src/report/core/sqs-report-calculator.service.ts b/src/report/core/sqs-report-calculator.service.ts index 3ea2743..3b89b98 100644 --- a/src/report/core/sqs-report-calculator.service.ts +++ b/src/report/core/sqs-report-calculator.service.ts @@ -8,7 +8,7 @@ import { ReportData } from '../../domain/report-data'; import { map, mergeAll, Observable, switchMap } from 'rxjs'; import { ReportCalculation } from '../../domain/report-calculation'; import { ReportingStorage } from '../storage/reporting-storage.service'; -import { CouchSqsClient } from '../../couchdb/couch-sqs.client'; +import { CouchSqsClient } from '../sqs/couch-sqs.client'; import { v4 as uuidv4 } from 'uuid'; import { Reference } from '../../domain/reference'; diff --git a/src/report/di/report-configuration.ts b/src/report/di/report-configuration.ts index 947af88..956de05 100644 --- a/src/report/di/report-configuration.ts +++ b/src/report/di/report-configuration.ts @@ -1,36 +1,52 @@ -import { - DefaultCouchDbClientFactory, - DefaultCouchSqsClientFactory, -} from '../../couchdb/default-factory'; +import { DefaultCouchDbClientFactory } from '../../couchdb/default-factory'; import { ConfigService } from '@nestjs/config'; -import { CouchSqsClient } from '../../couchdb/couch-sqs.client'; +import { CouchSqsClient, CouchSqsClientConfig } from '../sqs/couch-sqs.client'; import { ReportingStorage } from '../storage/reporting-storage.service'; import { ReportRepository } from '../repository/report-repository.service'; import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; import { CreateReportCalculationUseCase } from '../core/use-cases/create-report-calculation-use-case.service'; +import axios from 'axios'; +import { HttpService } from '@nestjs/axios'; export const ReportCouchSqsClientFactory = ( configService: ConfigService, -): CouchSqsClient => - DefaultCouchSqsClientFactory( - 'REPORT_COUCH_SQS_CLIENT_CONFIG_', - configService, - ); +): CouchSqsClient => { + const CONFIG_PREFIX = 'SQS_CLIENT_'; + + const config: CouchSqsClientConfig = { + BASE_URL: configService.getOrThrow(CONFIG_PREFIX + 'BASE_URL'), + BASIC_AUTH_USER: configService.getOrThrow( + CONFIG_PREFIX + 'BASIC_AUTH_USER', + ), + BASIC_AUTH_PASSWORD: configService.getOrThrow( + CONFIG_PREFIX + 'BASIC_AUTH_PASSWORD', + ), + SCHEMA_DESIGN_CONFIG: configService.getOrThrow( + CONFIG_PREFIX + 'SCHEMA_DESIGN_CONFIG', + ), + }; + + const axiosInstance = axios.create(); + + axiosInstance.defaults.baseURL = config.BASE_URL; + axiosInstance.defaults.headers['Authorization'] = `Basic ${Buffer.from( + `${config.BASIC_AUTH_USER}:${config.BASIC_AUTH_PASSWORD}`, + ).toString('base64')}`; + + return new CouchSqsClient(new HttpService(axiosInstance), config); +}; export const ReportingStorageFactory = ( configService: ConfigService, ): ReportingStorage => new ReportingStorage( new ReportRepository( - DefaultCouchDbClientFactory( - 'REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_', - configService, - ), + DefaultCouchDbClientFactory('COUCH_DB_CLIENT_REPORT_', configService), ), new ReportCalculationRepository( DefaultCouchDbClientFactory( - 'REPORT_COUCH_DB_CLIENT_CONFIG_REPORT_CALCULATION_', + 'COUCH_DB_CLIENT_REPORT_CALCULATION_', configService, ), ), diff --git a/src/report/report.module.ts b/src/report/report.module.ts index 20aed51..dae987b 100644 --- a/src/report/report.module.ts +++ b/src/report/report.module.ts @@ -7,7 +7,7 @@ import { ReportCalculationTask } from './tasks/report-calculation-task.service'; import { ReportCalculationProcessor } from './tasks/report-calculation-processor.service'; import { SqsReportCalculator } from './core/sqs-report-calculator.service'; import { CreateReportCalculationUseCase } from './core/use-cases/create-report-calculation-use-case.service'; -import { CouchSqsClient } from '../couchdb/couch-sqs.client'; +import { CouchSqsClient } from './sqs/couch-sqs.client'; import { ConfigService } from '@nestjs/config'; import { CreateReportCalculationUseCaseFactory, diff --git a/src/couchdb/couch-sqs-client.service.spec.ts b/src/report/sqs/couch-sqs-client.service.spec.ts similarity index 100% rename from src/couchdb/couch-sqs-client.service.spec.ts rename to src/report/sqs/couch-sqs-client.service.spec.ts diff --git a/src/couchdb/couch-sqs.client.ts b/src/report/sqs/couch-sqs.client.ts similarity index 100% rename from src/couchdb/couch-sqs.client.ts rename to src/report/sqs/couch-sqs.client.ts