From 055ea6973e07fc6f13b3aafd3cee91879ea4f345 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Tue, 13 Feb 2024 16:53:22 +0100 Subject: [PATCH] feat(report): implement report api endpoints (#2) closes #7 --- .github/workflows/pr-update.yml | 4 +- nest-cli.json | 4 +- package-lock.json | 126 ++++++---- package.json | 6 +- src/app.controller.spec.ts | 21 +- src/app.controller.ts | 21 +- src/app.module.ts | 13 +- src/couchdb/couch-db-client.service.spec.ts | 29 +++ src/couchdb/couch-db-client.service.ts | 121 +++++++++ src/couchdb/dtos.ts | 28 +++ src/domain/reference.ts | 20 ++ src/domain/report-calculation.ts | 63 +++++ src/domain/report-data.ts | 34 +++ src/domain/report.ts | 36 +++ src/main.ts | 14 -- src/query-body.dto.ts | 5 + src/report/controller/dtos.ts | 25 ++ .../report-calculation.controller.spec.ts | 43 ++++ .../report-calculation.controller.ts | 95 +++++++ .../controller/report.controller.spec.ts | 39 +++ src/report/controller/report.controller.ts | 49 ++++ src/report/core/report-calculator.ts | 7 + src/report/core/report-storage.ts | 30 +++ .../sqs-report-calculator.service.spec.ts | 18 ++ .../core/sqs-report-calculator.service.ts | 23 ++ src/report/report.module.ts | 26 ++ .../report-calculation-repository.service.ts | 231 ++++++++++++++++++ .../report-repository.service.spec.ts | 31 +++ .../repository/report-repository.service.ts | 98 ++++++++ .../storage/report-storage.service.spec.ts | 38 +++ src/report/storage/report-storage.service.ts | 147 +++++++++++ ...port-calculation-processor.service.spec.ts | 45 ++++ .../report-calculation-processor.service.ts | 95 +++++++ .../report-calculation-task.service.spec.ts | 43 ++++ .../tasks/report-calculation-task.service.ts | 26 ++ tsconfig.json | 4 +- 36 files changed, 1552 insertions(+), 106 deletions(-) create mode 100644 src/couchdb/couch-db-client.service.spec.ts create mode 100644 src/couchdb/couch-db-client.service.ts create mode 100644 src/couchdb/dtos.ts create mode 100644 src/domain/reference.ts create mode 100644 src/domain/report-calculation.ts create mode 100644 src/domain/report-data.ts create mode 100644 src/domain/report.ts create mode 100644 src/report/controller/dtos.ts create mode 100644 src/report/controller/report-calculation.controller.spec.ts create mode 100644 src/report/controller/report-calculation.controller.ts create mode 100644 src/report/controller/report.controller.spec.ts create mode 100644 src/report/controller/report.controller.ts create mode 100644 src/report/core/report-calculator.ts create mode 100644 src/report/core/report-storage.ts create mode 100644 src/report/core/sqs-report-calculator.service.spec.ts create mode 100644 src/report/core/sqs-report-calculator.service.ts create mode 100644 src/report/report.module.ts create mode 100644 src/report/repository/report-calculation-repository.service.ts create mode 100644 src/report/repository/report-repository.service.spec.ts create mode 100644 src/report/repository/report-repository.service.ts create mode 100644 src/report/storage/report-storage.service.spec.ts create mode 100644 src/report/storage/report-storage.service.ts create mode 100644 src/report/tasks/report-calculation-processor.service.spec.ts create mode 100644 src/report/tasks/report-calculation-processor.service.ts create mode 100644 src/report/tasks/report-calculation-task.service.spec.ts create mode 100644 src/report/tasks/report-calculation-task.service.ts diff --git a/.github/workflows/pr-update.yml b/.github/workflows/pr-update.yml index 911034f..6e7cd15 100644 --- a/.github/workflows/pr-update.yml +++ b/.github/workflows/pr-update.yml @@ -14,12 +14,14 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and test image + - name: Build, test and publish test image uses: docker/build-push-action@v3 with: context: ./ file: ./build/Dockerfile builder: ${{ steps.buildx.outputs.name }} + push: true + tags: aamdigital/query-ms:pr-${{ github.event.number }} cache-from: type=gha cache-to: type=gha,mode=max build-args: | diff --git a/nest-cli.json b/nest-cli.json index 6e477fd..d0091b8 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -1,7 +1,5 @@ { "collection": "@nestjs/schematics", "sourceRoot": "src", - "compilerOptions": { - "plugins": ["@nestjs/swagger"] - } + "compilerOptions": {} } diff --git a/package-lock.json b/package-lock.json index 4a03edc..da9f0f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "deployer-backend", + "name": "query-backend", "version": "0.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "deployer-backend", + "name": "query-backend", "version": "0.0.1", "license": "GPL-3.0", "dependencies": { @@ -14,12 +14,13 @@ "@nestjs/config": "^3.1.1", "@nestjs/core": "^9.3.10", "@nestjs/platform-express": "^9.3.10", - "@nestjs/swagger": "^6.3.0", + "@nestjs/schedule": "4.0.0", "@ntegral/nestjs-sentry": "^4.0.0", "@sentry/node": "^7.38.0", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", - "rxjs": "^7.8.0" + "rxjs": "^7.8.0", + "uuid": "9.0.1" }, "devDependencies": { "@nestjs/cli": "^9.2.0", @@ -29,6 +30,7 @@ "@types/jest": "^29.4.0", "@types/node": "^18.13.0", "@types/supertest": "^2.0.12", + "@types/uuid": "9.0.8", "@typescript-eslint/eslint-plugin": "^5.52.0", "@typescript-eslint/parser": "^5.52.0", "eslint": "^8.34.0", @@ -1653,6 +1655,14 @@ "reflect-metadata": "^0.1.13" } }, + "node_modules/@nestjs/config/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@nestjs/core": { "version": "9.4.3", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-9.4.3.tgz", @@ -1742,10 +1752,20 @@ "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==", "optional": true }, + "node_modules/@nestjs/graphql/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "optional": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@nestjs/mapped-types": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@nestjs/mapped-types/-/mapped-types-1.2.2.tgz", "integrity": "sha512-3dHxLXs3M0GPiriAcCFFJQHoDFUuzTD5w6JDhE7TyfT89YKpe6tcCCIqOZWdXmt9AZjjK30RkHRSFF+QEnWFQg==", + "optional": true, "peerDependencies": { "@nestjs/common": "^7.0.8 || ^8.0.0 || ^9.0.0", "class-transformer": "^0.2.0 || ^0.3.0 || ^0.4.0 || ^0.5.0", @@ -1781,6 +1801,20 @@ "@nestjs/core": "^9.0.0" } }, + "node_modules/@nestjs/schedule": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.0.0.tgz", + "integrity": "sha512-zz4h54m/F/1qyQKvMJCRphmuwGqJltDAkFxUXCVqJBXEs5kbPt93Pza3heCQOcMH22MZNhGlc9DmDMLXVHmgVQ==", + "dependencies": { + "cron": "3.1.3", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0", + "reflect-metadata": "^0.1.12" + } + }, "node_modules/@nestjs/schematics": { "version": "9.2.0", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.2.0.tgz", @@ -1796,37 +1830,6 @@ "typescript": ">=4.3.5" } }, - "node_modules/@nestjs/swagger": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/swagger/-/swagger-6.3.0.tgz", - "integrity": "sha512-Gnig189oa1tD+h0BYIfUwhp/wvvmTn6iO3csR2E4rQrDTgCxSxZDlNdfZo3AC+Rmf8u0KX4ZAX1RZN1qXTtC7A==", - "dependencies": { - "@nestjs/mapped-types": "1.2.2", - "js-yaml": "4.1.0", - "lodash": "4.17.21", - "path-to-regexp": "3.2.0", - "swagger-ui-dist": "4.18.2" - }, - "peerDependencies": { - "@fastify/static": "^6.0.0", - "@nestjs/common": "^9.0.0", - "@nestjs/core": "^9.0.0", - "class-transformer": "*", - "class-validator": "*", - "reflect-metadata": "^0.1.12" - }, - "peerDependenciesMeta": { - "@fastify/static": { - "optional": true - }, - "class-transformer": { - "optional": true - }, - "class-validator": { - "optional": true - } - } - }, "node_modules/@nestjs/testing": { "version": "9.4.3", "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-9.4.3.tgz", @@ -2222,6 +2225,11 @@ "integrity": "sha512-U3PUjAudAdJBeC2pgN8uTIKgxrb4nlDF3SF0++EldXQvQBGkpFZMSnwQiIoDU77tv45VgNkl/L4ouD+rEomujw==", "dev": true }, + "node_modules/@types/luxon": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.8.tgz", + "integrity": "sha512-jYvz8UMLDgy3a5SkGJne8H7VA7zPV2Lwohjx0V8V31+SqAjNmurWMkk9cQhfvlcnXWudBpK9xPM1n4rljOcHYQ==" + }, "node_modules/@types/mime": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.4.tgz", @@ -2304,6 +2312,12 @@ "@types/superagent": "*" } }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "dev": true + }, "node_modules/@types/yargs": { "version": "17.0.29", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.29.tgz", @@ -2852,7 +2866,8 @@ "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "node_modules/array-flatten": { "version": "1.1.1", @@ -3624,6 +3639,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.3.tgz", + "integrity": "sha512-KVxeKTKYj2eNzN4ElnT6nRSbjbfhyxR92O/Jdp6SH3pc05CDJws59jBrZWEMQlxevCiE6QUTrXy+Im3vC3oD3A==", + "dependencies": { + "@types/luxon": "~3.3.0", + "luxon": "~3.4.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -4534,9 +4558,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", "funding": [ { "type": "individual", @@ -5905,6 +5929,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, "dependencies": { "argparse": "^2.0.1" }, @@ -6095,6 +6120,14 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "engines": { + "node": ">=12" + } + }, "node_modules/macos-release": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.1.tgz", @@ -7714,11 +7747,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/swagger-ui-dist": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-4.18.2.tgz", - "integrity": "sha512-oVBoBl9Dg+VJw8uRWDxlyUyHoNEDC0c1ysT6+Boy6CTgr2rUcLcfPon4RvxgS2/taNW6O0+US+Z/dlAsWFjOAQ==" - }, "node_modules/symbol-observable": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-4.0.0.tgz", @@ -8224,9 +8252,13 @@ } }, "node_modules/uuid": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", - "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "bin": { "uuid": "dist/bin/uuid" } diff --git a/package.json b/package.json index de466b5..5b9de61 100644 --- a/package.json +++ b/package.json @@ -26,12 +26,13 @@ "@nestjs/config": "^3.1.1", "@nestjs/core": "^9.3.10", "@nestjs/platform-express": "^9.3.10", - "@nestjs/swagger": "^6.3.0", + "@nestjs/schedule": "4.0.0", "@ntegral/nestjs-sentry": "^4.0.0", "@sentry/node": "^7.38.0", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", - "rxjs": "^7.8.0" + "rxjs": "^7.8.0", + "uuid": "9.0.1" }, "devDependencies": { "@nestjs/cli": "^9.2.0", @@ -41,6 +42,7 @@ "@types/jest": "^29.4.0", "@types/node": "^18.13.0", "@types/supertest": "^2.0.12", + "@types/uuid": "9.0.8", "@typescript-eslint/eslint-plugin": "^5.52.0", "@typescript-eslint/parser": "^5.52.0", "eslint": "^8.34.0", diff --git a/src/app.controller.spec.ts b/src/app.controller.spec.ts index 6e265af..4303657 100644 --- a/src/app.controller.spec.ts +++ b/src/app.controller.spec.ts @@ -21,7 +21,7 @@ describe('AppController', () => { get: jest.fn().mockReturnValue(of({ data: undefined })), }; const mockConfigService = { - getOrThrow: (key) => { + getOrThrow: (key: any) => { switch (key) { case 'DATABASE_URL': return dbUrl; @@ -168,24 +168,7 @@ describe('AppController', () => { it('should throw error trying to query a non-sql report', (done) => { const report: SqlReport = { mode: 'exporting' as any, - aggregationDefinitions: undefined, - }; - mockHttp.get.mockReturnValue(of({ data: report })); - - controller - .queryData('ReportConfig:some-id', 'app', 'valid token') - .subscribe({ - error: (err) => { - expect(err).toBeInstanceOf(BadRequestException); - done(); - }, - }); - }); - - it('should throw sql query is not defined', (done) => { - const report: SqlReport = { - mode: 'sql', - aggregationDefinitions: undefined, + aggregationDefinitions: [], }; mockHttp.get.mockReturnValue(of({ data: report })); diff --git a/src/app.controller.ts b/src/app.controller.ts index ed282cd..d218116 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -9,8 +9,7 @@ import { } from '@nestjs/common'; import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; -import { ApiHeader, ApiOperation, ApiParam } from '@nestjs/swagger'; -import { catchError, concat, map, mergeMap, toArray } from 'rxjs'; +import { catchError, concat, map, mergeMap, Observable, toArray } from 'rxjs'; import { SqlReport } from './sql-report'; import { QueryBody } from './query-body.dto'; @@ -26,16 +25,6 @@ export class AppController { private configService: ConfigService, ) {} - @ApiOperation({ - description: `Get the results for the report with the given ID. User needs 'read' access for the requested report entity.`, - }) - @ApiParam({ name: 'id', description: '(full) ID of the report entity' }) - @ApiParam({ name: 'db', example: 'app', description: 'name of database' }) - @ApiHeader({ - name: 'Authorization', - required: false, - description: 'request needs to be authenticated', - }) @Post(':db/:id') queryData( @Param('id') reportId: string, @@ -73,11 +62,15 @@ export class AppController { ).pipe( // combine results of each request toArray(), - map((res) => [].concat(...res)), + map((res) => res.flat()), ); } - private getQueryResult(query: string, args: QueryBody, db: string) { + private getQueryResult( + query: string, + args: QueryBody | undefined, + db: string, + ): Observable { const data: SqsRequest = { query: query }; // There needs to be the same amount of "?" in the query as elements in "args" if (args?.from && args?.to && query.match(/\?/g)?.length === 2) { diff --git a/src/app.module.ts b/src/app.module.ts index 03589e0..2f5b0d5 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -3,8 +3,9 @@ import { SentryInterceptor, SentryModule } from '@ntegral/nestjs-sentry'; import { SeverityLevel } from '@sentry/types'; import { APP_INTERCEPTOR } from '@nestjs/core'; import { ConfigModule, ConfigService } from '@nestjs/config'; -import { AppController } from './app.controller'; import { HttpModule } from '@nestjs/axios'; +import { ReportModule } from './report/report.module'; +import { ScheduleModule } from '@nestjs/schedule'; const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; @@ -25,17 +26,18 @@ const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; ], imports: [ HttpModule, + ScheduleModule.forRoot(), ConfigModule.forRoot({ isGlobal: true }), SentryModule.forRootAsync({ imports: [ConfigModule], inject: [ConfigService], useFactory: async (configService: ConfigService) => { if (!configService.get('SENTRY_DSN')) { - return; + return {}; } return { - dsn: configService.get('SENTRY_DSN'), + dsn: configService.getOrThrow('SENTRY_DSN'), debug: true, environment: 'prod', release: 'backend@' + process.env.npm_package_version, @@ -47,7 +49,7 @@ const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; }, }, beforeSend: (event) => { - if (lowSeverityLevels.includes(event.level)) { + if (lowSeverityLevels.includes(event.level as SeverityLevel)) { return null; } else { return event; @@ -56,7 +58,8 @@ const lowSeverityLevels: SeverityLevel[] = ['log', 'info']; }; }, }), + ReportModule, ], - controllers: [AppController], + controllers: [], }) export class AppModule {} diff --git a/src/couchdb/couch-db-client.service.spec.ts b/src/couchdb/couch-db-client.service.spec.ts new file mode 100644 index 0000000..2f4ecba --- /dev/null +++ b/src/couchdb/couch-db-client.service.spec.ts @@ -0,0 +1,29 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { CouchDbClient } from './couch-db-client.service'; +import { HttpModule, HttpService } from '@nestjs/axios'; +import { of } from 'rxjs'; + +describe('CouchDbClient', () => { + let service: CouchDbClient; + + let mockHttp: { head: jest.Mock; post: jest.Mock; get: jest.Mock }; + + beforeEach(async () => { + mockHttp = { + head: jest.fn().mockReturnValue(of({ data: undefined })), + post: jest.fn().mockReturnValue(of({ data: undefined })), + get: jest.fn().mockReturnValue(of({ data: undefined })), + }; + + const module: TestingModule = await Test.createTestingModule({ + imports: [HttpModule], + providers: [CouchDbClient, { provide: HttpService, useValue: mockHttp }], + }).compile(); + + service = module.get(CouchDbClient); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/couchdb/couch-db-client.service.ts b/src/couchdb/couch-db-client.service.ts new file mode 100644 index 0000000..ea3d252 --- /dev/null +++ b/src/couchdb/couch-db-client.service.ts @@ -0,0 +1,121 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { catchError, map, Observable, of, switchMap } from 'rxjs'; +import { HttpService } from '@nestjs/axios'; +import { AxiosHeaders } from 'axios'; + +@Injectable() +export class CouchDbClient { + private readonly logger = new Logger(CouchDbClient.name); + + constructor(private httpService: HttpService) {} + + headDatabaseDocument( + databaseUrl: string, + databaseName: string, + documentId: string, + config?: any, + ) { + return this.httpService + .head(`${databaseUrl}/${databaseName}/${documentId}`, config) + .pipe( + catchError((err) => { + this.handleError(err); + throw err; + }), + ); + } + + getDatabaseDocument( + databaseUrl: string, + databaseName: string, + documentId: string, + config?: any, + ): Observable { + return this.httpService + .get(`${databaseUrl}/${databaseName}/${documentId}`, config) + .pipe( + map((response) => { + return response.data; + }), + catchError((err) => { + this.handleError(err); + throw err; + }), + ); + } + + find( + databaseUrl: string, + databaseName: string, + body: any, + config?: any, + ): Observable { + return this.httpService + .post(`${databaseUrl}/${databaseName}/_find`, body, config) + .pipe( + map((response) => { + return response.data; + }), + catchError((err) => { + this.handleError(err); + throw err; + }), + ); + } + + putDatabaseDocument( + databaseUrl: string, + databaseName: string, + documentId: string, + body: any, + config?: any, + ): Observable { + return this.latestRef(databaseUrl, databaseName, documentId, config).pipe( + switchMap((rev) => { + if (rev) { + config.headers['If-Match'] = rev; + } + + return this.httpService + .put(`${databaseUrl}/${databaseName}/${documentId}`, body, config) + .pipe( + map((response) => { + return response.data; + }), + catchError((err) => { + this.handleError(err); + throw err; + }), + ); + }), + ); + } + + private latestRef( + databaseUrl: string, + databaseName: string, + documentId: string, + config?: any, + ): Observable { + return this.headDatabaseDocument( + databaseUrl, + databaseName, + documentId, + config, + ).pipe( + map((response): string | undefined => { + const headers = response.headers; + if (headers instanceof AxiosHeaders && headers.has('etag')) { + return headers['etag'].replaceAll('"', ''); + } + }), + catchError((err) => { + return of(undefined); + }), + ); + } + + private handleError(err: any) { + this.logger.debug(err); + } +} diff --git a/src/couchdb/dtos.ts b/src/couchdb/dtos.ts new file mode 100644 index 0000000..dcacffe --- /dev/null +++ b/src/couchdb/dtos.ts @@ -0,0 +1,28 @@ +export interface CouchDbRow { + id: string; + key: string; + value: { + rev: string; + }; + doc: T; +} + +export class DocSuccess { + ok: boolean; + id: string; + rev: string; + + constructor(ok: boolean, id: string, rev: string) { + this.ok = ok; + this.id = id; + this.rev = rev; + } +} + +export class FindResponse { + constructor(docs: T[]) { + this.docs = docs; + } + + docs: T[]; +} diff --git a/src/domain/reference.ts b/src/domain/reference.ts new file mode 100644 index 0000000..3634645 --- /dev/null +++ b/src/domain/reference.ts @@ -0,0 +1,20 @@ +/** + * Representation of a reference to another Domain Object. + * Used, when just the Identifier is needed, not the hole object. + * + * @example: You want to trigger a calculation for new Report + * and just got the ReportId from your controller. You just pass a Reference to that Report: + * + * triggerCalculation(reportId: Reference): void {} + * + * const reportId = "r-1"; + * triggerCalculation(new Reference(reportId)); + * + */ +export class Reference { + constructor(id: string) { + this.id = id; + } + + id: string; +} diff --git a/src/domain/report-calculation.ts b/src/domain/report-calculation.ts new file mode 100644 index 0000000..92c9323 --- /dev/null +++ b/src/domain/report-calculation.ts @@ -0,0 +1,63 @@ +import { Reference } from './reference'; + +export enum ReportCalculationStatus { + PENDING = 'PENDING', + RUNNING = 'RUNNING', + FINISHED_SUCCESS = 'FINISHED_SUCCESS', + FINISHED_ERROR = 'FINISHED_ERROR', +} + +export type ReportCalculationOutcome = + | ReportCalculationOutcomeSuccess + | ReportCalculationOutcomeError; + +export interface ReportCalculationOutcomeSuccess { + result_hash: string; +} + +export interface ReportCalculationOutcomeError { + errorCode: string; + errorMessage: string; +} + +/** + * A ReportCalculation represents a calculation run for a specific Report. + * A Report can have multiple ReportCalculations. + */ +export class ReportCalculation { + id: string; + report: Reference; + status: ReportCalculationStatus; + start_date: string | null; + end_date: string | null; + outcome: ReportCalculationOutcome | null; + + constructor(id: string, report: Reference) { + this.id = id; + this.report = report; + this.status = ReportCalculationStatus.PENDING; + this.start_date = null; + this.end_date = null; + this.outcome = null; + } + + setStatus(status: ReportCalculationStatus): ReportCalculation { + this.status = status; + return this; + } + + setStartDate(startDate: string | null): ReportCalculation { + this.start_date = startDate; + return this; + } + + setEndDate(endDate: string | null): ReportCalculation { + this.end_date = endDate; + return this; + } + + setOutcome(outcome: ReportCalculationOutcome | null): ReportCalculation { + this.outcome = outcome; + return this; + } +} diff --git a/src/domain/report-data.ts b/src/domain/report-data.ts new file mode 100644 index 0000000..89c66b0 --- /dev/null +++ b/src/domain/report-data.ts @@ -0,0 +1,34 @@ +import { Reference } from './reference'; +import * as crypto from 'crypto'; + +/** + * The actual result of a ReportCalculation. + * + * @field data Final format is described by the ReportSchema + * linked in the related Report. + */ +export class ReportData { + id: string; + + constructor(id: string, report: Reference, calculation: Reference) { + this.id = id; + this.report = report; + this.calculation = calculation; + } + + report: Reference; + calculation: Reference; + data: any; + + setData(data: any): ReportData { + this.data = data; + return this; + } + + asHash(): string { + return crypto + .createHash('sha256') + .update(JSON.stringify(this)) + .digest('hex'); + } +} diff --git a/src/domain/report.ts b/src/domain/report.ts new file mode 100644 index 0000000..1d4102d --- /dev/null +++ b/src/domain/report.ts @@ -0,0 +1,36 @@ +/** + * Defines the expected format of a ReportData + */ +export interface ReportSchema { + fields: { [key: string]: any }; +} + +/** + * Representation of a user configured data export. + * The expected format of related ReportData will match with the ReportSchema. + */ +export class Report { + id: string; + name: string; + schema: ReportSchema | undefined; + + constructor(id: string, name: string) { + this.id = id; + this.name = name; + } + + setId(id: string): Report { + this.id = id; + return this; + } + + setName(name: string): Report { + this.name = name; + return this; + } + + setSchema(schema: ReportSchema): Report { + this.schema = schema; + return this; + } +} diff --git a/src/main.ts b/src/main.ts index e3acf97..ca8b82b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,24 +1,10 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './app.module'; -import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import { SentryService } from '@ntegral/nestjs-sentry'; async function bootstrap() { const app = await NestFactory.create(AppModule); - // SwaggerUI setup see https://docs.nestjs.com/openapi/introduction#bootstrap - const config = new DocumentBuilder() - .setTitle(process.env.npm_package_name) - .setDescription(process.env.npm_package_description) - .setVersion(process.env.npm_package_version) - .addBearerAuth(undefined, 'BearerAuth') - .addSecurityRequirements('BearerAuth') - .build(); - const document = SwaggerModule.createDocument(app, config); - SwaggerModule.setup('api', app, document, { - swaggerOptions: { persistAuthorization: true }, - }); - // Logging everything through sentry app.useLogger(SentryService.SentryServiceInstance()); diff --git a/src/query-body.dto.ts b/src/query-body.dto.ts index ea80508..0136960 100644 --- a/src/query-body.dto.ts +++ b/src/query-body.dto.ts @@ -6,4 +6,9 @@ export class QueryBody { from: string; to: string; + + constructor(from: string, to: string) { + this.from = from; + this.to = to; + } } diff --git a/src/report/controller/dtos.ts b/src/report/controller/dtos.ts new file mode 100644 index 0000000..4d778ba --- /dev/null +++ b/src/report/controller/dtos.ts @@ -0,0 +1,25 @@ +/** + * This is the interface shared to external users of the API endpoints. + */ +export class ReportDto { + constructor( + id: string, + name: string, + schema: any, + calculationPending: boolean | null = null, + ) { + this.id = id; + this.name = name; + this.calculationPending = calculationPending; + this.schema = { + fields: schema, + }; + } + + id: string; + name: string; + calculationPending: boolean | null; + schema: { + fields: any; + }; +} diff --git a/src/report/controller/report-calculation.controller.spec.ts b/src/report/controller/report-calculation.controller.spec.ts new file mode 100644 index 0000000..315363d --- /dev/null +++ b/src/report/controller/report-calculation.controller.spec.ts @@ -0,0 +1,43 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReportCalculationController } from './report-calculation.controller'; +import { DefaultReportStorage } from '../storage/report-storage.service'; +import { HttpModule } from '@nestjs/axios'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { ReportController } from './report.controller'; +import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; +import { ReportRepository } from '../repository/report-repository.service'; +import { ConfigService } from '@nestjs/config'; + +describe('ReportCalculationController', () => { + let controller: ReportCalculationController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [ReportCalculationController], + imports: [HttpModule], + providers: [ + CouchDbClient, + DefaultReportStorage, + ReportController, + ReportCalculationRepository, + ReportRepository, + { + provide: ConfigService, + useValue: { + getOrThrow: jest.fn((key) => { + return 'foo'; + }), + }, + }, + ], + }).compile(); + + controller = module.get( + ReportCalculationController, + ); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/src/report/controller/report-calculation.controller.ts b/src/report/controller/report-calculation.controller.ts new file mode 100644 index 0000000..5f72bf2 --- /dev/null +++ b/src/report/controller/report-calculation.controller.ts @@ -0,0 +1,95 @@ +import { + Controller, + Get, + Headers, + NotFoundException, + Param, + Post, +} from '@nestjs/common'; +import { DefaultReportStorage } from '../storage/report-storage.service'; +import { map, Observable, switchMap } from 'rxjs'; +import { ReportCalculation } from '../../domain/report-calculation'; +import { Reference } from '../../domain/reference'; +import { ReportData } from '../../domain/report-data'; +import { v4 as uuidv4 } from 'uuid'; + +@Controller('/api/v1/reporting') +export class ReportCalculationController { + constructor(private reportStorage: DefaultReportStorage) {} + + @Post('/report-calculation/report/:reportId') + startCalculation( + @Headers('Authorization') token: string, + @Param('reportId') reportId: string, + ): Observable { + return this.reportStorage.fetchReport(token, new Reference(reportId)).pipe( + switchMap((value) => { + if (!value) { + throw new NotFoundException(); + } + + return this.reportStorage + .storeCalculation( + new ReportCalculation( + `ReportCalculation:${uuidv4()}`, + new Reference(reportId), + ), + ) + .pipe( + map((reportCalculation) => new Reference(reportCalculation.id)), + ); + }), + ); + } + + @Get('/report-calculation/report/:reportId') + fetchReportCalculations( + @Headers('Authorization') token: string, + @Param('reportId') reportId: string, + ): Observable { + return this.reportStorage.fetchCalculations(new Reference(reportId)); + } + + @Get('/report-calculation/:calculationId') + fetchRun( + @Headers('Authorization') token: string, + @Param('calculationId') calculationId: string, + ): Observable { + return this.reportStorage + .fetchCalculation(new Reference(calculationId)) + .pipe( + switchMap((calculation) => { + if (!calculation) { + throw new NotFoundException(); + } + + return this.reportStorage + .fetchReport(token, new Reference(calculation.report.id)) + .pipe( + map((report) => { + if (!report) { + throw new NotFoundException(); + } + + return calculation; + }), + ); + }), + ); + } + + @Get('/report-calculation/:calculationId/data') + fetchRunData( + @Headers('Authorization') token: string, + @Param('calculationId') calculationId: string, + ): Observable { + return this.reportStorage.fetchData(new Reference(calculationId)).pipe( + map((value) => { + if (!value) { + throw new NotFoundException(); + } + return value; + }), + ); + } +} diff --git a/src/report/controller/report.controller.spec.ts b/src/report/controller/report.controller.spec.ts new file mode 100644 index 0000000..bdb63be --- /dev/null +++ b/src/report/controller/report.controller.spec.ts @@ -0,0 +1,39 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReportController } from './report.controller'; +import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportRepository } from '../repository/report-repository.service'; +import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; +import { HttpModule } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; + +describe('ReportController', () => { + let service: ReportController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [HttpModule], + providers: [ + CouchDbClient, + DefaultReportStorage, + ReportController, + ReportRepository, + ReportCalculationRepository, + { + provide: ConfigService, + useValue: { + getOrThrow: jest.fn((key) => { + return 'foo'; + }), + }, + }, + ], + }).compile(); + + service = module.get(ReportController); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/report/controller/report.controller.ts b/src/report/controller/report.controller.ts new file mode 100644 index 0000000..7f890c8 --- /dev/null +++ b/src/report/controller/report.controller.ts @@ -0,0 +1,49 @@ +import { Controller, Get, Headers, Param } from '@nestjs/common'; +import { + defaultIfEmpty, + map, + mergeMap, + Observable, + switchMap, + zipAll, +} from 'rxjs'; +import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ReportDto } from './dtos'; +import { Reference } from '../../domain/reference'; +import { Report } from '../../domain/report'; + +@Controller('/api/v1/reporting') +export class ReportController { + constructor(private reportStorage: DefaultReportStorage) {} + + @Get('/report') + fetchReports( + @Headers('Authorization') token: string, + ): Observable { + return this.reportStorage.fetchAllReports(token, 'sql').pipe( + mergeMap((reports) => reports.map((report) => this.getReportDto(report))), + zipAll(), + defaultIfEmpty([]), + ); + } + + @Get('/report/:reportId') + fetchReport( + @Headers('Authorization') token: string, + @Param('reportId') reportId: string, + ): Observable { + return this.reportStorage + .fetchReport(token, new Reference(reportId)) + .pipe(switchMap((report) => this.getReportDto(report))); + } + private getReportDto(report: Report): Observable { + return this.reportStorage + .isCalculationOngoing(new Reference(report.id)) + .pipe( + map( + (value) => + new ReportDto(report.id, report.name, report.schema, value), + ), + ); + } +} diff --git a/src/report/core/report-calculator.ts b/src/report/core/report-calculator.ts new file mode 100644 index 0000000..b60d5dd --- /dev/null +++ b/src/report/core/report-calculator.ts @@ -0,0 +1,7 @@ +import { ReportData } from '../../domain/report-data'; +import { Observable } from 'rxjs'; +import { ReportCalculation } from '../../domain/report-calculation'; + +export interface ReportCalculator { + calculate(reportCalculation: ReportCalculation): Observable; +} diff --git a/src/report/core/report-storage.ts b/src/report/core/report-storage.ts new file mode 100644 index 0000000..b969162 --- /dev/null +++ b/src/report/core/report-storage.ts @@ -0,0 +1,30 @@ +import { Reference } from '../../domain/reference'; +import { Report } from '../../domain/report'; +import { Observable } from 'rxjs'; +import { ReportCalculation } from '../../domain/report-calculation'; +import { ReportData } from '../../domain/report-data'; + +export interface ReportStorage { + fetchAllReports(authToken: string, mode: string): Observable; + + fetchReport( + authToken: string, + reportRef: Reference, + ): Observable; + + fetchPendingCalculations(): Observable; + + fetchCalculations(reportRef: Reference): Observable; + + fetchCalculation( + runRef: Reference, + ): Observable; + + storeCalculation(run: ReportCalculation): Observable; + + storeData(runData: ReportData): Observable; + + fetchData(runRef: Reference): Observable; + + isCalculationOngoing(reportRef: Reference): Observable; +} diff --git a/src/report/core/sqs-report-calculator.service.spec.ts b/src/report/core/sqs-report-calculator.service.spec.ts new file mode 100644 index 0000000..5bcaf9f --- /dev/null +++ b/src/report/core/sqs-report-calculator.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { SqsReportCalculator } from './sqs-report-calculator.service'; + +describe('SqsReportCalculatorService', () => { + let service: SqsReportCalculator; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [SqsReportCalculator], + }).compile(); + + service = module.get(SqsReportCalculator); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/report/core/sqs-report-calculator.service.ts b/src/report/core/sqs-report-calculator.service.ts new file mode 100644 index 0000000..76972db --- /dev/null +++ b/src/report/core/sqs-report-calculator.service.ts @@ -0,0 +1,23 @@ +import { Injectable } from '@nestjs/common'; +import { ReportCalculator } from './report-calculator'; +import { ReportData } from '../../domain/report-data'; +import { delay, Observable, of } from 'rxjs'; +import { ReportCalculation } from '../../domain/report-calculation'; +import { Reference } from '../../domain/reference'; +import { v4 as uuidv4 } from 'uuid'; + +@Injectable() +export class SqsReportCalculator implements ReportCalculator { + calculate(reportCalculation: ReportCalculation): Observable { + return of( + new ReportData( + `ReportData:${uuidv4()}`, + reportCalculation.report, + new Reference(reportCalculation.id), + ).setData({ + foo: 'bar', + dummyReportData: 'foo', + }), + ).pipe(delay(5000)); + } +} diff --git a/src/report/report.module.ts b/src/report/report.module.ts new file mode 100644 index 0000000..53cc2e0 --- /dev/null +++ b/src/report/report.module.ts @@ -0,0 +1,26 @@ +import { Module } from '@nestjs/common'; +import { DefaultReportStorage } from './storage/report-storage.service'; +import { ReportController } from './controller/report.controller'; +import { HttpModule } from '@nestjs/axios'; +import { ReportRepository } from './repository/report-repository.service'; +import { ReportCalculationRepository } from './repository/report-calculation-repository.service'; +import { ReportCalculationController } from './controller/report-calculation.controller'; +import { ReportCalculationTask } from './tasks/report-calculation-task.service'; +import { ReportCalculationProcessor } from './tasks/report-calculation-processor.service'; +import { SqsReportCalculator } from './core/sqs-report-calculator.service'; +import { CouchDbClient } from '../couchdb/couch-db-client.service'; + +@Module({ + controllers: [ReportController, ReportCalculationController], + imports: [HttpModule], + providers: [ + DefaultReportStorage, + ReportRepository, + ReportCalculationRepository, + ReportCalculationTask, + ReportCalculationProcessor, + SqsReportCalculator, + CouchDbClient, + ], +}) +export class ReportModule {} diff --git a/src/report/repository/report-calculation-repository.service.ts b/src/report/repository/report-calculation-repository.service.ts new file mode 100644 index 0000000..bc02554 --- /dev/null +++ b/src/report/repository/report-calculation-repository.service.ts @@ -0,0 +1,231 @@ +import { + ForbiddenException, + Injectable, + NotFoundException, + UnauthorizedException, +} from '@nestjs/common'; +import { ReportCalculation } from '../../domain/report-calculation'; +import { Reference } from '../../domain/reference'; +import { ReportData } from '../../domain/report-data'; +import { catchError, map, Observable, switchMap } from 'rxjs'; +import { ConfigService } from '@nestjs/config'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { DocSuccess, FindResponse } from '../../couchdb/dtos'; + +export interface ReportCalculationEntity { + id: string; + key: string; + value: { + rev: string; + }; + doc: ReportCalculation; +} + +export interface FetchReportCalculationsResponse { + total_rows: number; + offset: number; + rows: ReportCalculationEntity[]; +} + +@Injectable() +export class ReportCalculationRepository { + static readonly REPORT_DATABASE_URL = 'REPORT_DATABASE_URL'; + static readonly REPORT_DATABASE_NAME = 'REPORT_DATABASE_NAME'; + + readonly databaseUrl: string; + readonly databaseName: string; + + readonly authHeaderValue: string; + + constructor( + private couchDbClient: CouchDbClient, + private configService: ConfigService, + ) { + this.databaseUrl = this.configService.getOrThrow( + ReportCalculationRepository.REPORT_DATABASE_URL, + ); + this.databaseName = this.configService.getOrThrow( + ReportCalculationRepository.REPORT_DATABASE_NAME, + ); + + const authHeader = Buffer.from( + `${this.configService.getOrThrow( + 'DATABASE_USER', + )}:${this.configService.getOrThrow('DATABASE_PASSWORD')}`, + ).toString('base64'); + this.authHeaderValue = `Basic ${authHeader}`; + } + + storeCalculation( + reportCalculation: ReportCalculation, + ): Observable { + return this.couchDbClient.putDatabaseDocument( + this.databaseUrl, + this.databaseName, + reportCalculation.id, + reportCalculation, + { + headers: { + Authorization: this.authHeaderValue, + }, + }, + ); + } + + fetchCalculations(): Observable { + return this.couchDbClient.getDatabaseDocument( + this.databaseUrl, + this.databaseName, + '_all_docs', + { + params: { + include_docs: true, + start_key: '"ReportCalculation"', + end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character + }, + headers: { + Authorization: this.authHeaderValue, + }, + }, + ); + } + + fetchCalculation( + calculationRef: Reference, + ): Observable { + return this.couchDbClient + .getDatabaseDocument( + this.databaseUrl, + this.databaseName, + calculationRef.id, + { + headers: { + Authorization: this.authHeaderValue, + }, + }, + ) + .pipe( + map((rawReportCalculation) => + new ReportCalculation( + rawReportCalculation.id, + rawReportCalculation.report, + ) + .setStatus(rawReportCalculation.status) + .setStartDate(rawReportCalculation.start_date) + .setEndDate(rawReportCalculation.end_date) + .setOutcome(rawReportCalculation.outcome), + ), + catchError((err, caught) => { + if (err.response.status === 404) { + throw new NotFoundException(); + } + throw err; + }), + ); + } + + storeData(data: ReportData): Observable { + return this.couchDbClient + .putDatabaseDocument(this.databaseUrl, this.databaseName, data.id, data, { + headers: { + Authorization: this.authHeaderValue, + }, + }) + .pipe( + switchMap(() => this.fetchCalculation(data.calculation)), + switchMap((calculation) => { + if (!calculation) { + throw new NotFoundException(); + } + + calculation.setOutcome({ + result_hash: data.asHash(), + }); + + return this.couchDbClient + .putDatabaseDocument( + this.databaseUrl, + this.databaseName, + calculation.id, + calculation, + { + headers: { + Authorization: this.authHeaderValue, + }, + }, + ) + .pipe(map(() => data)); + }), + ); + } + + fetchData(calculationRef: Reference): Observable { + return this.fetchCalculation(calculationRef).pipe( + map((calculation) => { + if (!calculation) { + throw new NotFoundException(); + } + + return calculation.id; + }), + switchMap((calculationId) => { + this.couchDbClient.getDatabaseDocument( + this.databaseUrl, + this.databaseName, + '_all_docs', + { + params: { + include_docs: true, + start_key: '"' + calculationId + '"', + end_key: '"ReportCalculation' + '\ufff0"', // ufff0 -> high value unicode character + }, + headers: { + Authorization: this.authHeaderValue, + }, + }, + ); + + return this.couchDbClient + .find>( + this.databaseUrl, + this.databaseName, + { + selector: { + 'calculation.id': { $eq: calculationId }, + }, + }, + { + headers: { + Authorization: this.authHeaderValue, + }, + }, + ) + .pipe( + map((value) => { + if (value.docs && value.docs.length === 1) { + return value.docs[0]; + } else { + throw new NotFoundException(); + } + }), + catchError((err) => { + this.handleError(err); + throw err; + }), + ); + }), + ); + } + + private handleError(err: any) { + if (err.response.status === 401) { + throw new UnauthorizedException(); + } + if (err.response.status === 403) { + throw new ForbiddenException(); + } + if (err.response.status === 404) { + throw new NotFoundException(); + } + } +} diff --git a/src/report/repository/report-repository.service.spec.ts b/src/report/repository/report-repository.service.spec.ts new file mode 100644 index 0000000..da7b40f --- /dev/null +++ b/src/report/repository/report-repository.service.spec.ts @@ -0,0 +1,31 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReportRepository } from './report-repository.service'; +import { HttpModule } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; + +describe('ReportRepositoryService', () => { + let service: ReportRepository; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [HttpModule], + providers: [ + ReportRepository, + { + provide: ConfigService, + useValue: { + getOrThrow: jest.fn((key) => { + return 'foo'; + }), + }, + }, + ], + }).compile(); + + service = module.get(ReportRepository); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/report/repository/report-repository.service.ts b/src/report/repository/report-repository.service.ts new file mode 100644 index 0000000..d4955cc --- /dev/null +++ b/src/report/repository/report-repository.service.ts @@ -0,0 +1,98 @@ +import { + ForbiddenException, + Injectable, + NotFoundException, + UnauthorizedException, +} from '@nestjs/common'; +import { HttpService } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; +import { catchError, map, Observable } from 'rxjs'; +import { CouchDbRow } from '../../couchdb/dtos'; + +interface ReportDoc { + _id: string; + _rev: string; + title: string; + mode: string; + aggregationDefinitions: string[]; + created: { + at: string; + by: string; + }; + updated: { + at: string; + by: string; + }; +} + +interface FetchReportsResponse { + total_rows: number; + offset: number; + rows: CouchDbRow[]; +} + +@Injectable() +export class ReportRepository { + private dbUrl: string = this.configService.getOrThrow('DATABASE_URL'); + private databaseUser: string = this.configService.getOrThrow('DATABASE_USER'); + private databasePassword: string = + this.configService.getOrThrow('DATABASE_PASSWORD'); + + private authHeaderValue: string; + + constructor(private http: HttpService, private configService: ConfigService) { + const authHeader = Buffer.from( + `${this.databaseUser}:${this.databasePassword}`, + ).toString('base64'); + this.authHeaderValue = `Basic ${authHeader}`; + } + + fetchReports(authToken: string): Observable { + return this.http + .get(`${this.dbUrl}/app/_all_docs`, { + params: { + include_docs: true, + startkey: '"ReportConfig:"', + endkey: '"ReportConfig:' + '\ufff0"', + }, + headers: { + Authorization: authToken, + }, + }) + .pipe( + map((value) => value.data), + catchError((err, caught) => { + this.handleError(err); + throw caught; + }), + ); + } + + fetchReport(authToken: string, reportId: string): Observable { + return this.http + .get(`${this.dbUrl}/app/${reportId}`, { + headers: { + Authorization: authToken, + }, + }) + .pipe( + map((value) => value.data), + catchError((err, caught) => { + this.handleError(err); + throw caught; + }), + ); + } + + private handleError(err: any) { + if (err.response.status === 401) { + throw new UnauthorizedException(); + } + if (err.response.status === 403) { + throw new ForbiddenException(); + } + if (err.response.status === 404) { + throw new NotFoundException(); + } + } +} diff --git a/src/report/storage/report-storage.service.spec.ts b/src/report/storage/report-storage.service.spec.ts new file mode 100644 index 0000000..b88428d --- /dev/null +++ b/src/report/storage/report-storage.service.spec.ts @@ -0,0 +1,38 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReportStorage } from '../core/report-storage'; +import { DefaultReportStorage } from './report-storage.service'; +import { ReportRepository } from '../repository/report-repository.service'; +import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; +import { HttpModule } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; + +describe('DefaultReportStorage', () => { + let service: ReportStorage; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [HttpModule], + providers: [ + DefaultReportStorage, + ReportRepository, + ReportCalculationRepository, + CouchDbClient, + { + provide: ConfigService, + useValue: { + getOrThrow: jest.fn((key) => { + return 'foo'; + }), + }, + }, + ], + }).compile(); + + service = module.get(DefaultReportStorage); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/report/storage/report-storage.service.ts b/src/report/storage/report-storage.service.ts new file mode 100644 index 0000000..d885bbe --- /dev/null +++ b/src/report/storage/report-storage.service.ts @@ -0,0 +1,147 @@ +import { Reference } from '../../domain/reference'; +import { Report } from '../../domain/report'; +import { ReportStorage } from '../core/report-storage'; +import { ReportRepository } from '../repository/report-repository.service'; +import { map, Observable, switchMap } from 'rxjs'; +import { Injectable, NotFoundException } from '@nestjs/common'; +import { + ReportCalculation, + ReportCalculationStatus, +} from '../../domain/report-calculation'; +import { + ReportCalculationEntity, + ReportCalculationRepository, +} from '../repository/report-calculation-repository.service'; +import { ReportData } from '../../domain/report-data'; + +@Injectable() +export class DefaultReportStorage implements ReportStorage { + constructor( + private reportRepository: ReportRepository, + private reportCalculationRepository: ReportCalculationRepository, + ) {} + + fetchAllReports(authToken: string, mode = 'sql'): Observable { + return this.reportRepository.fetchReports(authToken).pipe( + map((response) => { + if (!response || !response.rows) { + return []; + } + + return response.rows + .filter((row) => row.doc.mode === mode) + .map((reportEntity) => + new Report(reportEntity.id, reportEntity.doc.title).setSchema({ + fields: reportEntity.doc.aggregationDefinitions, // todo generate actual fields here + }), + ); + }), + ); + } + + fetchReport(authToken: string, reportRef: Reference): Observable { + return this.reportRepository.fetchReport(authToken, reportRef.id).pipe( + map((reportDoc) => { + return new Report(reportDoc._id, reportDoc.title).setSchema({ + fields: reportDoc.aggregationDefinitions, // todo generate actual fields here + }); + }), + ); + } + + isCalculationOngoing(reportRef: Reference): Observable { + return this.reportCalculationRepository + .fetchCalculations() + .pipe( + map( + (response) => + response.rows + .filter( + (reportCalculation) => + reportCalculation.doc.report.id === reportRef.id, + ) + .filter( + (reportCalculation) => + reportCalculation.doc.status === + ReportCalculationStatus.PENDING || + reportCalculation.doc.status === + ReportCalculationStatus.RUNNING, + ).length > 0, + ), + ); + } + + fetchPendingCalculations(): Observable { + return this.reportCalculationRepository + .fetchCalculations() + .pipe( + map((response) => + response.rows + .filter( + (reportCalculation) => + reportCalculation.doc.status === + ReportCalculationStatus.PENDING, + ) + .map((entity: ReportCalculationEntity) => + this.mapFromEntity(entity), + ), + ), + ); + } + + fetchCalculations(reportRef: Reference): Observable { + return this.reportCalculationRepository + .fetchCalculations() + .pipe( + map((response) => + response.rows + .filter( + (reportCalculation) => + reportCalculation.doc.report.id === reportRef.id, + ) + .map((entity: ReportCalculationEntity) => + this.mapFromEntity(entity), + ), + ), + ); + } + + fetchCalculation( + calculationRef: Reference, + ): Observable { + return this.reportCalculationRepository.fetchCalculation(calculationRef); + } + + storeCalculation( + reportCalculation: ReportCalculation, + ): Observable { + return this.reportCalculationRepository + .storeCalculation(reportCalculation) + .pipe( + switchMap((entity) => this.fetchCalculation(new Reference(entity.id))), + map((value) => { + if (!value) { + throw new NotFoundException(); + } else { + return value; + } + }), + ); + } + + storeData(reportData: ReportData): Observable { + return this.reportCalculationRepository.storeData(reportData); + } + + fetchData(calculationRef: Reference): Observable { + return this.reportCalculationRepository.fetchData(calculationRef); + } + + private mapFromEntity(entity: ReportCalculationEntity): ReportCalculation { + return new ReportCalculation(entity.doc.id, entity.doc.report) + .setStatus(entity.doc.status) + .setStartDate(entity.doc.start_date) + .setEndDate(entity.doc.end_date) + .setOutcome(entity.doc.outcome); + } +} diff --git a/src/report/tasks/report-calculation-processor.service.spec.ts b/src/report/tasks/report-calculation-processor.service.spec.ts new file mode 100644 index 0000000..12efd8c --- /dev/null +++ b/src/report/tasks/report-calculation-processor.service.spec.ts @@ -0,0 +1,45 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReportCalculationProcessor } from './report-calculation-processor.service'; +import { DefaultReportStorage } from '../storage/report-storage.service'; +import { HttpModule } from '@nestjs/axios'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; +import { ReportCalculationTask } from './report-calculation-task.service'; +import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; +import { ReportRepository } from '../repository/report-repository.service'; +import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; +import { ConfigService } from '@nestjs/config'; + +describe('ReportCalculationProcessorService', () => { + let service: ReportCalculationProcessor; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [HttpModule], + providers: [ + CouchDbClient, + ReportCalculationTask, + DefaultReportStorage, + ReportCalculationProcessor, + SqsReportCalculator, + ReportRepository, + ReportCalculationRepository, + { + provide: ConfigService, + useValue: { + getOrThrow: jest.fn((key) => { + return 'foo'; + }), + }, + }, + ], + }).compile(); + + service = module.get( + ReportCalculationProcessor, + ); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/report/tasks/report-calculation-processor.service.ts b/src/report/tasks/report-calculation-processor.service.ts new file mode 100644 index 0000000..4383142 --- /dev/null +++ b/src/report/tasks/report-calculation-processor.service.ts @@ -0,0 +1,95 @@ +import { Injectable } from '@nestjs/common'; +import { catchError, map, Observable, of, switchMap } from 'rxjs'; +import { + ReportCalculation, + ReportCalculationStatus, +} from '../../domain/report-calculation'; +import { DefaultReportStorage } from '../storage/report-storage.service'; +import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; +import { ReportData } from '../../domain/report-data'; + +@Injectable() +export class ReportCalculationProcessor { + constructor( + private reportStorage: DefaultReportStorage, + private reportCalculator: SqsReportCalculator, + ) {} + + processNextPendingCalculation(): Observable { + return this.reportStorage.fetchPendingCalculations().pipe( + switchMap((calculations) => { + const next = calculations.pop(); + + if (!next) { + return of(); + } + + return this.reportStorage + .storeCalculation( + next + .setStatus(ReportCalculationStatus.RUNNING) + .setStartDate(new Date().toISOString()), + ) + .pipe( + switchMap((reportCalculation) => + this.reportCalculator.calculate(reportCalculation).pipe( + switchMap((reportData) => + this.reportStorage + .storeData(reportData) + .pipe( + switchMap(() => + this.markCalculationAsFinishedSuccess( + reportCalculation, + reportData, + ), + ), + ) + .pipe(switchMap(() => of())), + ), + ), + ), + catchError((err) => + this.markCalculationAsFinishedError(next, err).pipe( + map(() => { + throw err; + }), + ), + ), + ); + }), + catchError((err, caught) => { + console.log(err); + return of(); + }), + ); + } + + private markCalculationAsFinishedSuccess( + reportCalculation: ReportCalculation, + reportData: ReportData, + ): Observable { + return this.reportStorage.storeCalculation( + reportCalculation + .setStatus(ReportCalculationStatus.FINISHED_SUCCESS) + .setOutcome({ + result_hash: reportData.asHash(), + }) + .setEndDate(new Date().toISOString()), + ); + } + + private markCalculationAsFinishedError( + reportCalculation: ReportCalculation, + err: any, + ): Observable { + return this.reportStorage.storeCalculation( + reportCalculation + .setStatus(ReportCalculationStatus.FINISHED_ERROR) + .setOutcome({ + errorCode: 'CALCULATION_FAILED', + errorMessage: err, + }) + .setEndDate(new Date().toISOString()), + ); + } +} diff --git a/src/report/tasks/report-calculation-task.service.spec.ts b/src/report/tasks/report-calculation-task.service.spec.ts new file mode 100644 index 0000000..5a92b87 --- /dev/null +++ b/src/report/tasks/report-calculation-task.service.spec.ts @@ -0,0 +1,43 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ReportCalculationTask } from './report-calculation-task.service'; +import { ReportCalculationProcessor } from './report-calculation-processor.service'; +import { SqsReportCalculator } from '../core/sqs-report-calculator.service'; +import { DefaultReportStorage } from '../storage/report-storage.service'; +import { ConfigService } from '@nestjs/config'; +import { ReportRepository } from '../repository/report-repository.service'; +import { ReportCalculationRepository } from '../repository/report-calculation-repository.service'; +import { HttpModule } from '@nestjs/axios'; +import { CouchDbClient } from '../../couchdb/couch-db-client.service'; + +describe('ReportCalculationTaskService', () => { + let service: ReportCalculationTask; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [HttpModule], + providers: [ + CouchDbClient, + ReportCalculationTask, + DefaultReportStorage, + ReportCalculationProcessor, + SqsReportCalculator, + ReportRepository, + ReportCalculationRepository, + { + provide: ConfigService, + useValue: { + getOrThrow: jest.fn((key) => { + return 'foo'; + }), + }, + }, + ], + }).compile(); + + service = module.get(ReportCalculationTask); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/report/tasks/report-calculation-task.service.ts b/src/report/tasks/report-calculation-task.service.ts new file mode 100644 index 0000000..7ecc0b2 --- /dev/null +++ b/src/report/tasks/report-calculation-task.service.ts @@ -0,0 +1,26 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { ReportCalculationProcessor } from './report-calculation-processor.service'; +import { catchError } from 'rxjs'; + +@Injectable() +export class ReportCalculationTask { + private readonly logger = new Logger(ReportCalculationTask.name); + + constructor(private reportCalculationProcessor: ReportCalculationProcessor) {} + + @Cron(CronExpression.EVERY_10_SECONDS) + handleCron(): void { + this.reportCalculationProcessor + .processNextPendingCalculation() + .pipe( + catchError((err, caught) => { + this.logger.log('reportCalculationProcessor', err, caught); + throw err; + }), + ) + .subscribe((_) => { + this.logger.log('done'); + }); + } +} diff --git a/tsconfig.json b/tsconfig.json index 36810b5..8603582 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,8 @@ "baseUrl": "./", "incremental": true, "skipLibCheck": true, - "resolveJsonModule": true + "resolveJsonModule": true, + "strict": true, + "alwaysStrict": true } }