From c86abde3c1b0c778c10a44f106dc944e40131f62 Mon Sep 17 00:00:00 2001 From: Tom Winter Date: Tue, 23 Jul 2024 16:24:14 +0200 Subject: [PATCH] feat: date range and big data support for sql reports (#17) --- .../aambackendservice/Application.kt | 3 + .../couchdb/core/CouchDbClient.kt | 196 +++---------- .../couchdb/core/CouchDbStorage.kt | 46 --- .../couchdb/core/DefaultCouchDbClient.kt | 273 ++++++++++++++++++ .../couchdb/di/CouchDbConfiguration.kt | 4 +- .../couchdb/dto/CouchDbDto.kt | 9 + .../aambackendservice/domain/EntityType.kt | 5 + .../core/CouchDbDatabaseChangeDetection.kt | 8 +- .../DefaultCreateDocumentChangeUseCase.kt | 10 +- .../changes/di/ChangesConfiguration.kt | 10 +- .../reporting/domain/Report.kt | 1 + .../reporting/domain/ReportCalculation.kt | 39 +-- .../reporting/domain/ReportData.kt | 23 -- .../DefaultAddWebhookSubscriptionUseCase.kt | 47 +-- .../core/DefaultTriggerWebhookUseCase.kt | 8 +- .../notification/core/NotificationService.kt | 4 +- .../notification/storage/WebhookRepository.kt | 13 +- .../report/controller/ReportController.kt | 6 +- .../DefaultIdentifyAffectedReportsUseCase.kt | 8 +- .../core/DefaultReportCalculationProcessor.kt | 51 ++-- ...efaultReportDocumentChangeEventConsumer.kt | 10 +- .../reporting/report/core/QueryStorage.kt | 6 +- .../reporting/report/core/ReportingStorage.kt | 16 +- .../report/di/ReportConfiguration.kt | 5 +- .../reporting/report/dto/ControllerDtos.kt | 1 + .../report/jobs/ReportCalculationJob.kt | 2 +- .../reporting/report/sqs/SqsQueryStorage.kt | 39 +-- .../reporting/report/sqs/SqsSchemaService.kt | 85 +++++- .../controller/ReportCalculationController.kt | 142 ++++++--- .../core/CreateReportCalculationUseCase.kt | 3 +- .../DefaultCreateReportCalculationUseCase.kt | 4 +- .../DefaultReportCalculationChangeUseCase.kt | 9 +- .../core/DefaultReportCalculator.kt | 66 +++-- .../core/ReportCalculator.kt | 3 +- .../reportcalculation/dto/ControllerDtos.kt | 29 ++ .../reporting/storage/DefaultReportStorage.kt | 77 +++++ .../storage/DefaultReportingStorage.kt | 70 +---- .../storage/ReportCalculationRepository.kt | 81 ++---- .../reporting/storage/ReportRepository.kt | 36 --- .../src/main/resources/application.yaml | 3 +- .../common/CouchDbTestingService.kt | 25 ++ .../e2e/CucumberIntegrationTest.kt | 12 +- .../src/test/resources/application-e2e.yaml | 2 + .../fetch-calculation-data.feature | 7 +- .../fetch-report-calculation.feature | 2 +- .../start-report-calculation.feature | 18 ++ .../documents/Config:CONFIG_ENTITY.json | 4 + .../documents/ReportCalculation:1.json | 9 +- .../documents/ReportCalculation:2.json | 7 +- .../documents/ReportCalculation:3.json | 13 + .../database/documents/ReportConfig:1.json | 1 + .../database/documents/ReportConfig:2.json | 1 + .../database/documents/ReportConfig:3.json | 10 + .../database/documents/ReportData:1.json | 22 +- docs/api-specs/reporting-api-v1.yaml | 48 ++- 55 files changed, 999 insertions(+), 633 deletions(-) delete mode 100644 application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbStorage.kt create mode 100644 application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/DefaultCouchDbClient.kt delete mode 100644 application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportData.kt create mode 100644 application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/dto/ControllerDtos.kt create mode 100644 application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportStorage.kt delete mode 100644 application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportRepository.kt create mode 100644 application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:3.json create mode 100644 application/aam-backend-service/src/test/resources/database/documents/ReportConfig:3.json diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/Application.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/Application.kt index 50e5d68..dcc464d 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/Application.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/Application.kt @@ -6,6 +6,7 @@ import org.springframework.boot.runApplication import org.springframework.scheduling.annotation.EnableScheduling import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RestController +import java.util.* @SpringBootApplication @ConfigurationPropertiesScan @@ -13,6 +14,8 @@ import org.springframework.web.bind.annotation.RestController class Application fun main(args: Array) { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")) + runApplication(*args) } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbClient.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbClient.kt index 1ef0ce2..24c98cb 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbClient.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbClient.kt @@ -3,187 +3,65 @@ package com.aamdigital.aambackendservice.couchdb.core import com.aamdigital.aambackendservice.couchdb.dto.CouchDbChangesResponse import com.aamdigital.aambackendservice.couchdb.dto.DocSuccess import com.aamdigital.aambackendservice.couchdb.dto.FindResponse -import com.aamdigital.aambackendservice.error.InternalServerException -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.node.ObjectNode -import org.slf4j.LoggerFactory -import org.springframework.core.ParameterizedTypeReference +import org.springframework.core.io.buffer.DataBuffer import org.springframework.http.HttpHeaders -import org.springframework.http.MediaType import org.springframework.util.MultiValueMap -import org.springframework.web.reactive.function.BodyInserters -import org.springframework.web.reactive.function.client.ClientResponse -import org.springframework.web.reactive.function.client.WebClient +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import kotlin.reflect.KClass -class CouchDbClient( - private val webClient: WebClient, - private val objectMapper: ObjectMapper -) : CouchDbStorage { +interface CouchDbClient { + fun allDatabases(): Mono> + fun changes(database: String, queryParams: MultiValueMap): Mono - private val logger = LoggerFactory.getLogger(javaClass) - - companion object { - private const val CHANGES_URL = "/_changes" - private const val FIND_URL = "/_find" - } - - override fun allDatabases(): Mono> { - return webClient - .get() - .uri("/_all_dbs") - .accept(MediaType.APPLICATION_JSON) - .exchangeToMono { response -> - response.bodyToMono(object : ParameterizedTypeReference>() {}) - } - } - - override fun changes( - database: String, queryParams: MultiValueMap - ): Mono { - return webClient.get().uri { - it.path("/$database/$CHANGES_URL") - it.queryParams(queryParams) - it.build() - }.accept(MediaType.APPLICATION_JSON).exchangeToMono { response -> - response.bodyToMono(CouchDbChangesResponse::class.java).mapNotNull { - it - } - } - } - - override fun find( - database: String, body: Map, queryParams: MultiValueMap, kClass: KClass - ): Mono> { - return webClient.post().uri { - it.path("/$database/$FIND_URL") - it.queryParams(queryParams) - it.build() - }.contentType(MediaType.APPLICATION_JSON).body(BodyInserters.fromValue(body)).exchangeToMono { - it.bodyToMono(ObjectNode::class.java).map { objectNode -> - val data = - (objectMapper.convertValue(objectNode, Map::class.java)["docs"] as Iterable<*>).map { entry -> - objectMapper.convertValue(entry, kClass.java) - } - - FindResponse(docs = data) - } - } - } + fun find( + database: String, + body: Map, + queryParams: MultiValueMap = getEmptyQueryParams(), + kClass: KClass + ): Mono> - override fun headDatabaseDocument( + fun headDatabaseDocument( database: String, documentId: String, - ): Mono { - return webClient - .head() - .uri { - it.path("/$database/$documentId") - it.build() - } - .accept(MediaType.APPLICATION_JSON).exchangeToMono { - if (it.statusCode().is2xxSuccessful) { - Mono.just(it.headers().asHttpHeaders()) - } else if (it.statusCode().is4xxClientError) { - Mono.just(HttpHeaders()) - } else { - throw InternalServerException() - } - } - } + ): Mono - override fun getDatabaseDocument( + fun getDatabaseDocument( database: String, documentId: String, - queryParams: MultiValueMap, + queryParams: MultiValueMap = getEmptyQueryParams(), kClass: KClass, - ): Mono { - return webClient.get().uri { - it.path("/$database/$documentId") - it.queryParams(queryParams) - it.build() - }.accept(MediaType.APPLICATION_JSON).exchangeToMono { - handleResponse(it, kClass) - } - } + ): Mono - override fun putDatabaseDocument( + fun putDatabaseDocument( database: String, documentId: String, - body: Any, - ): Mono { - return headDatabaseDocument( - database = database, - documentId = documentId - ).flatMap { httpHeaders -> - val etag = httpHeaders.eTag?.replace("\"", "") - - webClient.put().uri { - it.path("/$database/$documentId") - it.build() - }.body(BodyInserters.fromValue(body)).headers { - if (etag.isNullOrBlank().not()) { - it.set("If-Match", etag) - } - }.accept(MediaType.APPLICATION_JSON).exchangeToMono { - handleResponse(it, DocSuccess::class) - } - } - } + body: Any + ): Mono - override fun getPreviousDocRev( + fun getPreviousDocRev( database: String, documentId: String, rev: String, kClass: KClass, - ): Mono { - val allRevsInfoQueryParams = getEmptyQueryParams() - allRevsInfoQueryParams.set("revs_info", "true") - - return getDatabaseDocument( - database = database, - documentId = documentId, - queryParams = allRevsInfoQueryParams, - kClass = ObjectNode::class - ).flatMap { currentDoc -> - val revInfo = currentDoc.get("_revs_info") ?: return@flatMap Mono.empty() - - if (!revInfo.isArray) { - return@flatMap Mono.empty() - } - - val revIndex = revInfo.indexOfFirst { jsonNode -> jsonNode.get("rev").textValue().equals(rev) } + ): Mono - if (revIndex == -1) { - return@flatMap Mono.empty() - } - - if (revIndex + 1 >= revInfo.size()) { - return@flatMap Mono.empty() - } - - val previousRef = revInfo.get(revIndex + 1).get("rev").textValue() - - val previousRevQueryParams = getEmptyQueryParams() - previousRevQueryParams.set("rev", previousRef) + fun headAttachment( + database: String, + documentId: String, + attachmentId: String, + ): Mono - getDatabaseDocument( - database = database, - documentId = documentId, - queryParams = previousRevQueryParams, - kClass = ObjectNode::class - ).map { previousDoc -> - objectMapper.convertValue(previousDoc, kClass.java) - } - } - } + fun getAttachment( + database: String, + documentId: String, + attachmentId: String, + ): Flux - private fun handleResponse( - response: ClientResponse, typeReference: KClass - ): Mono { - return response.bodyToMono(typeReference.java).mapNotNull { - it - } - } + fun putAttachment( + database: String, + documentId: String, + attachmentId: String, + file: Flux + ): Mono } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbStorage.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbStorage.kt deleted file mode 100644 index 40f4bbb..0000000 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/CouchDbStorage.kt +++ /dev/null @@ -1,46 +0,0 @@ -package com.aamdigital.aambackendservice.couchdb.core - -import com.aamdigital.aambackendservice.couchdb.dto.CouchDbChangesResponse -import com.aamdigital.aambackendservice.couchdb.dto.DocSuccess -import com.aamdigital.aambackendservice.couchdb.dto.FindResponse -import org.springframework.http.HttpHeaders -import org.springframework.util.MultiValueMap -import reactor.core.publisher.Mono -import kotlin.reflect.KClass - -interface CouchDbStorage { - fun allDatabases(): Mono> - fun changes(database: String, queryParams: MultiValueMap): Mono - - fun find( - database: String, - body: Map, - queryParams: MultiValueMap, - kClass: KClass - ): Mono> - - fun headDatabaseDocument( - database: String, - documentId: String, - ): Mono - - fun getDatabaseDocument( - database: String, - documentId: String, - queryParams: MultiValueMap, - kClass: KClass, - ): Mono - - fun putDatabaseDocument( - database: String, - documentId: String, - body: Any - ): Mono - - fun getPreviousDocRev( - database: String, - documentId: String, - rev: String, - kClass: KClass, - ): Mono -} diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/DefaultCouchDbClient.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/DefaultCouchDbClient.kt new file mode 100644 index 0000000..b64784f --- /dev/null +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/core/DefaultCouchDbClient.kt @@ -0,0 +1,273 @@ +package com.aamdigital.aambackendservice.couchdb.core + +import com.aamdigital.aambackendservice.couchdb.dto.CouchDbChangesResponse +import com.aamdigital.aambackendservice.couchdb.dto.DocSuccess +import com.aamdigital.aambackendservice.couchdb.dto.FindResponse +import com.aamdigital.aambackendservice.error.InternalServerException +import com.aamdigital.aambackendservice.error.NotFoundException +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import org.slf4j.LoggerFactory +import org.springframework.core.ParameterizedTypeReference +import org.springframework.core.io.buffer.DataBuffer +import org.springframework.http.HttpHeaders +import org.springframework.http.MediaType +import org.springframework.util.MultiValueMap +import org.springframework.web.reactive.function.BodyInserters +import org.springframework.web.reactive.function.client.ClientResponse +import org.springframework.web.reactive.function.client.WebClient +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import kotlin.reflect.KClass + +class DefaultCouchDbClient( + private val webClient: WebClient, + private val objectMapper: ObjectMapper +) : CouchDbClient { + + private val logger = LoggerFactory.getLogger(javaClass) + + companion object { + private const val CHANGES_URL = "/_changes" + private const val FIND_URL = "/_find" + } + + override fun allDatabases(): Mono> { + return webClient + .get() + .uri("/_all_dbs") + .accept(MediaType.APPLICATION_JSON) + .exchangeToMono { response -> + response.bodyToMono(object : ParameterizedTypeReference>() {}) + } + } + + override fun changes( + database: String, queryParams: MultiValueMap + ): Mono { + return webClient.get().uri { + it.path("/$database/$CHANGES_URL") + it.queryParams(queryParams) + it.build() + }.accept(MediaType.APPLICATION_JSON).exchangeToMono { response -> + response.bodyToMono(CouchDbChangesResponse::class.java).mapNotNull { + it + } + } + } + + override fun find( + database: String, body: Map, queryParams: MultiValueMap, kClass: KClass + ): Mono> { + return webClient.post().uri { + it.path("/$database/$FIND_URL") + it.queryParams(queryParams) + it.build() + }.contentType(MediaType.APPLICATION_JSON).body(BodyInserters.fromValue(body)).exchangeToMono { + it.bodyToMono(ObjectNode::class.java).map { objectNode -> + val data = + (objectMapper.convertValue(objectNode, Map::class.java)["docs"] as Iterable<*>).map { entry -> + objectMapper.convertValue(entry, kClass.java) + } + + FindResponse(docs = data) + } + } + } + + override fun headDatabaseDocument( + database: String, + documentId: String, + ): Mono { + return webClient + .head() + .uri { + it.path("/$database/$documentId") + it.build() + } + .accept(MediaType.APPLICATION_JSON).exchangeToMono { + if (it.statusCode().is2xxSuccessful) { + Mono.just(it.headers().asHttpHeaders()) + } else if (it.statusCode().is4xxClientError) { + Mono.just(HttpHeaders()) + } else { + throw InternalServerException() + } + } + } + + override fun getDatabaseDocument( + database: String, + documentId: String, + queryParams: MultiValueMap, + kClass: KClass, + ): Mono { + return webClient.get().uri { + it.path("/$database/$documentId") + it.queryParams(queryParams) + it.build() + }.accept(MediaType.APPLICATION_JSON).exchangeToMono { + handleResponse(it, kClass) + } + } + + override fun putDatabaseDocument( + database: String, + documentId: String, + body: Any, + ): Mono { + return headDatabaseDocument( + database = database, + documentId = documentId + ).flatMap { httpHeaders -> + val etag = httpHeaders.eTag?.replace("\"", "") + + webClient.put() + .uri { + it.path("/$database/$documentId") + it.build() + } + .body(BodyInserters.fromValue(body)) + .headers { + if (etag.isNullOrBlank().not()) { + it.set("If-Match", etag) + } + } + .accept(MediaType.APPLICATION_JSON).exchangeToMono { + handleResponse(it, DocSuccess::class) + } + } + } + + override fun getPreviousDocRev( + database: String, + documentId: String, + rev: String, + kClass: KClass, + ): Mono { + val allRevsInfoQueryParams = getEmptyQueryParams() + allRevsInfoQueryParams.set("revs_info", "true") + + return getDatabaseDocument( + database = database, + documentId = documentId, + queryParams = allRevsInfoQueryParams, + kClass = ObjectNode::class + ).flatMap { currentDoc -> + val revInfo = currentDoc.get("_revs_info") ?: return@flatMap Mono.empty() + + if (!revInfo.isArray) { + return@flatMap Mono.empty() + } + + val revIndex = revInfo.indexOfFirst { jsonNode -> jsonNode.get("rev").textValue().equals(rev) } + + if (revIndex == -1) { + return@flatMap Mono.empty() + } + + if (revIndex + 1 >= revInfo.size()) { + return@flatMap Mono.empty() + } + + val previousRef = revInfo.get(revIndex + 1).get("rev").textValue() + + val previousRevQueryParams = getEmptyQueryParams() + previousRevQueryParams.set("rev", previousRef) + + getDatabaseDocument( + database = database, + documentId = documentId, + queryParams = previousRevQueryParams, + kClass = ObjectNode::class + ).map { previousDoc -> + objectMapper.convertValue(previousDoc, kClass.java) + } + } + } + + override fun getAttachment( + database: String, + documentId: String, + attachmentId: String, + ): Flux { + return webClient.get() + .uri { + it.path("$database/$documentId/$attachmentId") + it.build() + } + .accept(MediaType.APPLICATION_OCTET_STREAM) + .retrieve() + .onStatus({ it.is4xxClientError }, { + Mono.error(NotFoundException("Could not find attachment: $database/$documentId/$attachmentId")) + }) + .bodyToFlux(DataBuffer::class.java) + .doOnError { + logger.warn(it.localizedMessage, it) + } + } + + override fun headAttachment( + database: String, + documentId: String, + attachmentId: String, + ): Mono { + return webClient.head() + .uri { + it.path("$database/$documentId/$attachmentId") + it.build() + } + .accept(MediaType.APPLICATION_JSON).exchangeToMono { + if (it.statusCode().is2xxSuccessful) { + Mono.just(it.headers().asHttpHeaders()) + } else if (it.statusCode().is4xxClientError) { + Mono.just(HttpHeaders()) + } else { + throw InternalServerException() + } + } + } + + override fun putAttachment( + database: String, + documentId: String, + attachmentId: String, + file: Flux + ): Mono { + return headDatabaseDocument( + database = database, + documentId = documentId, + ) + .flatMap { httpHeaders -> + val etag = httpHeaders.eTag?.replace("\"", "") + webClient.put() + .uri { + it.path("$database/$documentId/$attachmentId") + it.build() + } + .body(BodyInserters.fromDataBuffers(file)) + .headers { + if (etag.isNullOrBlank().not()) { + it.set("If-Match", etag) + } + it.contentType = MediaType.APPLICATION_JSON + } + .retrieve() + .bodyToMono(DocSuccess::class.java) + .doOnSuccess { + logger.trace("[CouchDbClient] PUT Attachment response: {}", it) + } + .doOnError { + logger.error("[CouchDbClient] PUT Attachment failed: {}", it.localizedMessage) + } + } + } + + private fun handleResponse( + response: ClientResponse, typeReference: KClass + ): Mono { + return response.bodyToMono(typeReference.java).mapNotNull { + it + } + } +} diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/di/CouchDbConfiguration.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/di/CouchDbConfiguration.kt index 4d2ee8c..90221df 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/di/CouchDbConfiguration.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/di/CouchDbConfiguration.kt @@ -1,7 +1,7 @@ package com.aamdigital.aambackendservice.couchdb.di import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage +import com.aamdigital.aambackendservice.couchdb.core.DefaultCouchDbClient import com.fasterxml.jackson.databind.ObjectMapper import org.springframework.beans.factory.annotation.Qualifier import org.springframework.boot.context.properties.ConfigurationProperties @@ -18,7 +18,7 @@ class CouchDbConfiguration { fun defaultCouchDbStorage( @Qualifier("couch-db-client") webClient: WebClient, objectMapper: ObjectMapper, - ): CouchDbStorage = CouchDbClient(webClient, objectMapper) + ): CouchDbClient = DefaultCouchDbClient(webClient, objectMapper) @Bean(name = ["couch-db-client"]) fun couchDbWebClient(configuration: CouchDbClientConfiguration): WebClient { diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/dto/CouchDbDto.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/dto/CouchDbDto.kt index 124b37f..15e6310 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/dto/CouchDbDto.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/dto/CouchDbDto.kt @@ -20,6 +20,15 @@ data class DocSuccess( val rev: String, ) +data class AttachmentMetaData( + @JsonProperty("content_type") + val contentType: String, + val revpos: Int, + val digest: String, + val length: Long, + val stub: Boolean, +) + data class FindResponse( val docs: List ) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/domain/EntityType.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/domain/EntityType.kt index eb75480..d91c386 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/domain/EntityType.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/domain/EntityType.kt @@ -2,6 +2,11 @@ package com.aamdigital.aambackendservice.domain data class EntityAttribute( val name: String, + val type: EntityAttributeType, +) + +data class EntityAttributeType( + val field: String, val type: String, ) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/CouchDbDatabaseChangeDetection.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/CouchDbDatabaseChangeDetection.kt index c270aa9..858f6b3 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/CouchDbDatabaseChangeDetection.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/CouchDbDatabaseChangeDetection.kt @@ -1,6 +1,6 @@ package com.aamdigital.aambackendservice.reporting.changes.core -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient import com.aamdigital.aambackendservice.couchdb.core.getEmptyQueryParams import com.aamdigital.aambackendservice.reporting.changes.di.ChangesQueueConfiguration.Companion.DB_CHANGES_QUEUE import com.aamdigital.aambackendservice.reporting.changes.repository.SyncEntry @@ -11,7 +11,7 @@ import reactor.core.publisher.Mono import java.util.* class CouchDbDatabaseChangeDetection( - private val couchDbStorage: CouchDbStorage, + private val couchDbClient: CouchDbClient, private val documentChangeEventPublisher: ChangeEventPublisher, private val syncRepository: SyncRepository, ) : DatabaseChangeDetection { @@ -27,7 +27,7 @@ class CouchDbDatabaseChangeDetection( */ override fun checkForChanges(): Mono { logger.trace("[CouchDatabaseChangeDetection] start couchdb change detection...") - return couchDbStorage.allDatabases().flatMap { databases -> + return couchDbClient.allDatabases().flatMap { databases -> val requests = databases.filter { !it.startsWith("_") }.map { database -> fetchChangesForDatabase(database) } @@ -55,7 +55,7 @@ class CouchDbDatabaseChangeDetection( queryParams.set("limit", CHANGES_LIMIT.toString()) queryParams.set("include_docs", "true") - couchDbStorage.changes( + couchDbClient.changes( database = database, queryParams = queryParams ) } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/DefaultCreateDocumentChangeUseCase.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/DefaultCreateDocumentChangeUseCase.kt index 56aa469..0cf377a 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/DefaultCreateDocumentChangeUseCase.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/core/DefaultCreateDocumentChangeUseCase.kt @@ -1,6 +1,6 @@ package com.aamdigital.aambackendservice.reporting.changes.core -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient import com.aamdigital.aambackendservice.couchdb.core.getEmptyQueryParams import com.aamdigital.aambackendservice.reporting.changes.di.ChangesQueueConfiguration.Companion.DOCUMENT_CHANGES_EXCHANGE import com.aamdigital.aambackendservice.reporting.domain.event.DatabaseChangeEvent @@ -14,17 +14,17 @@ import reactor.core.publisher.Mono * Use case is called if a change on any database document is detected. */ class DefaultCreateDocumentChangeUseCase( - private val couchDbStorage: CouchDbStorage, + private val couchDbClient: CouchDbClient, private val objectMapper: ObjectMapper, private val documentChangeEventPublisher: ChangeEventPublisher, ) : CreateDocumentChangeUseCase { - val logger = LoggerFactory.getLogger(javaClass) + private val logger = LoggerFactory.getLogger(javaClass) override fun createEvent(event: DatabaseChangeEvent): Mono { val queryParams = getEmptyQueryParams() queryParams.set("rev", event.rev) - return couchDbStorage.getDatabaseDocument( + return couchDbClient.getDatabaseDocument( database = event.database, documentId = event.documentId, queryParams = queryParams, @@ -33,7 +33,7 @@ class DefaultCreateDocumentChangeUseCase( if (event.rev.isNullOrBlank()) { return Mono.empty() } else { - couchDbStorage.getPreviousDocRev( + couchDbClient.getPreviousDocRev( database = event.database, documentId = event.documentId, rev = event.rev, diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/di/ChangesConfiguration.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/di/ChangesConfiguration.kt index 5815502..68c1033 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/di/ChangesConfiguration.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/changes/di/ChangesConfiguration.kt @@ -1,6 +1,6 @@ package com.aamdigital.aambackendservice.reporting.changes.di -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient import com.aamdigital.aambackendservice.reporting.changes.core.ChangeEventPublisher import com.aamdigital.aambackendservice.reporting.changes.core.CouchDbDatabaseChangeDetection import com.aamdigital.aambackendservice.reporting.changes.core.CreateDocumentChangeUseCase @@ -31,23 +31,23 @@ class ChangesConfiguration { matchIfMissing = true ) fun couchDatabaseChangeDetection( - couchDbStorage: CouchDbStorage, + couchDbClient: CouchDbClient, changeEventPublisher: ChangeEventPublisher, syncRepository: SyncRepository, ): DatabaseChangeDetection = CouchDbDatabaseChangeDetection( - couchDbStorage, + couchDbClient, changeEventPublisher, syncRepository ) @Bean fun defaultAnalyseDocumentChangeUseCase( - couchDbStorage: CouchDbStorage, + couchDbClient: CouchDbClient, objectMapper: ObjectMapper, changeEventPublisher: ChangeEventPublisher ): CreateDocumentChangeUseCase = DefaultCreateDocumentChangeUseCase( - couchDbStorage = couchDbStorage, + couchDbClient = couchDbClient, objectMapper = objectMapper, documentChangeEventPublisher = changeEventPublisher, ) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/Report.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/Report.kt index 7c13522..f1847b9 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/Report.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/Report.kt @@ -6,4 +6,5 @@ data class Report( val mode: String?, val schema: ReportSchema?, val query: String, + val neededArgs: List = emptyList(), ) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportCalculation.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportCalculation.kt index 04b1d51..d217e8b 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportCalculation.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportCalculation.kt @@ -1,35 +1,20 @@ package com.aamdigital.aambackendservice.reporting.domain +import com.aamdigital.aambackendservice.couchdb.dto.AttachmentMetaData import com.aamdigital.aambackendservice.domain.DomainReference import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.annotation.JsonSubTypes -import com.fasterxml.jackson.annotation.JsonTypeInfo - -@JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) -@JsonSubTypes( - JsonSubTypes.Type(value = ReportCalculationOutcome.Success::class), - JsonSubTypes.Type(value = ReportCalculationOutcome.Failure::class), -) -sealed class ReportCalculationOutcome { - data class Success( - @JsonProperty("result_hash") - val resultHash: String, - ) : ReportCalculationOutcome() - - data class Failure( - val errorCode: String, - val errorMessage: String, - ) : ReportCalculationOutcome() -} data class ReportCalculation( @JsonProperty("_id") val id: String, val report: DomainReference, var status: ReportCalculationStatus, - var startDate: String? = null, - var endDate: String? = null, - var outcome: ReportCalculationOutcome? = null, + var errorDetails: String? = null, + var calculationStarted: String? = null, + var calculationCompleted: String? = null, + var args: MutableMap = mutableMapOf(), + @JsonProperty("_attachments") + val attachments: MutableMap = mutableMapOf(), ) { fun setStatus(status: ReportCalculationStatus): ReportCalculation { this.status = status @@ -37,17 +22,17 @@ data class ReportCalculation( } fun setStartDate(startDate: String?): ReportCalculation { - this.startDate = startDate + this.calculationStarted = startDate return this } - fun setEndDate(endDate: String?): ReportCalculation { - this.endDate = endDate + fun setErrorDetails(errorDetails: String?): ReportCalculation { + this.errorDetails = errorDetails return this } - fun setOutcome(outcome: ReportCalculationOutcome?): ReportCalculation { - this.outcome = outcome + fun setEndDate(endDate: String?): ReportCalculation { + this.calculationCompleted = endDate return this } } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportData.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportData.kt deleted file mode 100644 index c23ddca..0000000 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportData.kt +++ /dev/null @@ -1,23 +0,0 @@ -package com.aamdigital.aambackendservice.reporting.domain - -import com.aamdigital.aambackendservice.domain.DomainReference -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.ObjectMapper -import java.security.MessageDigest - -data class ReportData( - @JsonProperty("_id") - val id: String, - val report: DomainReference, - val calculation: DomainReference, - var data: List<*>, -) { - @OptIn(ExperimentalStdlibApi::class) - fun getDataHash(): String { - val mapper = ObjectMapper() - val md = MessageDigest.getInstance("SHA-256") - val input = mapper.writeValueAsString(data).toByteArray() - val bytes = md.digest(input) - return bytes.toHexString() - } -} diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultAddWebhookSubscriptionUseCase.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultAddWebhookSubscriptionUseCase.kt index 0cdaaf2..0873ccc 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultAddWebhookSubscriptionUseCase.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultAddWebhookSubscriptionUseCase.kt @@ -1,6 +1,7 @@ package com.aamdigital.aambackendservice.reporting.notification.core import com.aamdigital.aambackendservice.domain.DomainReference +import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationRequest import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationUseCase @@ -19,27 +20,35 @@ class DefaultAddWebhookSubscriptionUseCase( ).flatMap { reportingStorage.fetchCalculations( reportReference = report - ).flatMap { calculations -> - if (calculations.isEmpty()) { - createReportCalculationUseCase.startReportCalculation( - CreateReportCalculationRequest( - report = report, - ) - ).flatMap { - Mono.just(Unit) - } - } else { - notificationService.triggerWebhook( - report = report, - webhook = webhook, - reportCalculation = DomainReference( - calculations.sortedByDescending { it.endDate }.first().id - ) - ) - Mono.just(Unit) - } + ).flatMap { reportCalculations -> + handleReportCalculations(reportCalculations, report, webhook) } } + } + private fun handleReportCalculations( + calculations: List, + report: DomainReference, + webhook: DomainReference + ): Mono { + return if (calculations.isEmpty()) { + createReportCalculationUseCase.createReportCalculation( + CreateReportCalculationRequest( + report = report, + args = mutableMapOf() + ) + ).flatMap { + Mono.just(Unit) + } + } else { + notificationService.triggerWebhook( + report = report, + webhook = webhook, + reportCalculation = DomainReference( + calculations.sortedByDescending { it.calculationCompleted }.first().id + ) + ) + Mono.just(Unit) + } } } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultTriggerWebhookUseCase.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultTriggerWebhookUseCase.kt index cbf221e..d02e92d 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultTriggerWebhookUseCase.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/DefaultTriggerWebhookUseCase.kt @@ -11,6 +11,9 @@ import org.springframework.web.reactive.function.client.WebClient import reactor.core.publisher.Mono import java.net.URI +/** + * Calls a configured (external) webhook + */ class DefaultTriggerWebhookUseCase( private val notificationStorage: NotificationStorage, private val webClient: WebClient, @@ -55,8 +58,9 @@ class DefaultTriggerWebhookUseCase( response.bodyToMono(String::class.java) } .map { - logger.trace( - "[DefaultTriggerWebhookUseCase] Webhook trigger completed for Webhook: {} Report: {} Calculation: {} - Response: {}", + logger.debug( + "[DefaultTriggerWebhookUseCase] Webhook trigger completed for Webhook:" + + " {} Report: {} Calculation: {} - Response: {}", notificationEvent.webhookId, notificationEvent.reportId, notificationEvent.calculationId, diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/NotificationService.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/NotificationService.kt index a42d491..f387002 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/NotificationService.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/core/NotificationService.kt @@ -28,7 +28,7 @@ class NotificationService( } fun sendNotifications(report: DomainReference, reportCalculation: DomainReference): Mono { - logger.trace("[NotificationService]: Trigger all affected webhooks for ${report.id}") + logger.debug("[NotificationService]: Trigger all affected webhooks for ${report.id}") return getAffectedWebhooks(report) .map { webhooks -> webhooks.map { webhook -> @@ -42,7 +42,7 @@ class NotificationService( } fun triggerWebhook(report: DomainReference, reportCalculation: DomainReference, webhook: DomainReference) { - logger.trace("[NotificationService]: Trigger NotificationEvent for ${webhook.id} and ${report.id}") + logger.debug("[NotificationService]: Trigger NotificationEvent for ${webhook.id} and ${report.id}") notificationEventPublisher.publish( NotificationQueueConfiguration.NOTIFICATION_QUEUE, NotificationEvent( diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/storage/WebhookRepository.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/storage/WebhookRepository.kt index a341950..4b1f50b 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/storage/WebhookRepository.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/notification/storage/WebhookRepository.kt @@ -1,30 +1,27 @@ package com.aamdigital.aambackendservice.reporting.notification.storage -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient import com.aamdigital.aambackendservice.couchdb.core.getQueryParamsAllDocs import com.aamdigital.aambackendservice.couchdb.dto.DocSuccess import com.aamdigital.aambackendservice.domain.DomainReference import com.aamdigital.aambackendservice.error.InternalServerException import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode -import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import org.springframework.util.LinkedMultiValueMap import reactor.core.publisher.Mono @Service class WebhookRepository( - private val couchDbStorage: CouchDbStorage, + private val couchDbClient: CouchDbClient, private val objectMapper: ObjectMapper, ) { - private val logger = LoggerFactory.getLogger(javaClass) - companion object { private const val WEBHOOK_DATABASE = "notification-webhook" } fun fetchAllWebhooks(): Mono> { - return couchDbStorage + return couchDbClient .getDatabaseDocument( database = WEBHOOK_DATABASE, documentId = "_all_docs", @@ -46,7 +43,7 @@ class WebhookRepository( } fun fetchWebhook(webhookRef: DomainReference): Mono { - return couchDbStorage + return couchDbClient .getDatabaseDocument( database = WEBHOOK_DATABASE, documentId = webhookRef.id, @@ -56,7 +53,7 @@ class WebhookRepository( } fun storeWebhook(webhook: WebhookEntity): Mono { - return couchDbStorage + return couchDbClient .putDatabaseDocument( database = WEBHOOK_DATABASE, documentId = webhook.id, diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/controller/ReportController.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/controller/ReportController.kt index 9a3decf..ee8e575 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/controller/ReportController.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/controller/ReportController.kt @@ -4,6 +4,7 @@ import com.aamdigital.aambackendservice.domain.DomainReference import com.aamdigital.aambackendservice.error.NotFoundException import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage import com.aamdigital.aambackendservice.reporting.report.dto.ReportDto +import com.aamdigital.aambackendservice.reporting.storage.DefaultReportStorage import org.springframework.validation.annotation.Validated import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable @@ -16,10 +17,11 @@ import reactor.core.publisher.Mono @Validated class ReportController( private val reportingStorage: ReportingStorage, + private val reportStorage: DefaultReportStorage, ) { @GetMapping fun fetchReports(): Mono> { - return reportingStorage.fetchAllReports("sql") + return reportStorage.fetchAllReports("sql") .zipWith( reportingStorage.fetchPendingCalculations() ).map { results -> @@ -42,7 +44,7 @@ class ReportController( fun fetchReport( @PathVariable reportId: String ): Mono { - return reportingStorage + return reportStorage .fetchReport(DomainReference(id = reportId)) .zipWith( reportingStorage.fetchPendingCalculations() diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultIdentifyAffectedReportsUseCase.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultIdentifyAffectedReportsUseCase.kt index 3d629f3..d7b3656 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultIdentifyAffectedReportsUseCase.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultIdentifyAffectedReportsUseCase.kt @@ -2,21 +2,19 @@ package com.aamdigital.aambackendservice.reporting.report.core import com.aamdigital.aambackendservice.domain.DomainReference import com.aamdigital.aambackendservice.reporting.domain.event.DocumentChangeEvent -import org.slf4j.LoggerFactory +import com.aamdigital.aambackendservice.reporting.storage.DefaultReportStorage import reactor.core.publisher.Mono class DefaultIdentifyAffectedReportsUseCase( - private val reportingStorage: ReportingStorage, + private val reportStorage: DefaultReportStorage, private val reportSchemaGenerator: ReportSchemaGenerator, ) : IdentifyAffectedReportsUseCase { - val logger = LoggerFactory.getLogger(javaClass) - override fun analyse(documentChangeEvent: DocumentChangeEvent): Mono> { val changedEntity = documentChangeEvent.documentId.split(":").first() - return reportingStorage.fetchAllReports("sql") + return reportStorage.fetchAllReports("sql") .map { reports -> val affectedReports: MutableList = mutableListOf() reports.forEach { report -> diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportCalculationProcessor.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportCalculationProcessor.kt index 47e87ba..4fab11c 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportCalculationProcessor.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportCalculationProcessor.kt @@ -1,7 +1,8 @@ package com.aamdigital.aambackendservice.reporting.report.core +import com.aamdigital.aambackendservice.domain.DomainReference +import com.aamdigital.aambackendservice.error.NotFoundException import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation -import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationOutcome import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationStatus import com.aamdigital.aambackendservice.reporting.reportcalculation.core.ReportCalculator import reactor.core.publisher.Mono @@ -14,7 +15,7 @@ class DefaultReportCalculationProcessor( private val reportCalculator: ReportCalculator, ) : ReportCalculationProcessor { override fun processNextPendingCalculation(): Mono { - var calculation: ReportCalculation; + var calculation: ReportCalculation return reportingStorage.fetchPendingCalculations() .flatMap { calculations -> calculation = calculations.firstOrNull() @@ -30,35 +31,35 @@ class DefaultReportCalculationProcessor( .flatMap { reportCalculator.calculate(reportCalculation = it) } - .flatMap { reportData -> - reportingStorage.storeData( - reportData - ) + .flatMap { + reportingStorage.fetchCalculation(DomainReference(calculation.id)) } - .flatMap { reportData -> + .flatMap { updatedCalculation -> reportingStorage.storeCalculation( - reportCalculation = calculation - .setStatus(ReportCalculationStatus.FINISHED_SUCCESS) - .setOutcome( - ReportCalculationOutcome.Success( - resultHash = reportData.getDataHash() - ) + reportCalculation = updatedCalculation.orElseThrow { + NotFoundException( + "[DefaultReportCalculationProcessor]" + + " updated Calculation not available after reportCalculator.calculate()" ) + } + .setStatus(ReportCalculationStatus.FINISHED_SUCCESS) .setEndDate(getIsoLocalDateTime()) ).map {} } - .onErrorResume { - reportingStorage.storeCalculation( - reportCalculation = calculation - .setStatus(ReportCalculationStatus.FINISHED_ERROR) - .setOutcome( - ReportCalculationOutcome.Failure( - errorCode = "CALCULATION_FAILED", - errorMessage = it.localizedMessage, - ) - ) - .setEndDate(getIsoLocalDateTime()) - ).map {} + .onErrorResume { error -> + /* + We should think about moving the "prefetch" inside the ReportCalculationStorage, + instead of manually think about this every time. The "prefetch" ensures, + that the latest calculation is edited + */ + reportingStorage.fetchCalculation(DomainReference(calculation.id)).flatMap { + reportingStorage.storeCalculation( + reportCalculation = calculation + .setStatus(ReportCalculationStatus.FINISHED_ERROR) + .setErrorDetails(error.localizedMessage) + .setEndDate(getIsoLocalDateTime()) + ).map {} + } } } } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportDocumentChangeEventConsumer.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportDocumentChangeEventConsumer.kt index 56f015a..b278333 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportDocumentChangeEventConsumer.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/DefaultReportDocumentChangeEventConsumer.kt @@ -49,9 +49,10 @@ class DefaultReportDocumentChangeEventConsumer( val reportRef = payload.currentVersion["_id"] as String - return createReportCalculationUseCase.startReportCalculation( + return createReportCalculationUseCase.createReportCalculation( request = CreateReportCalculationRequest( - report = DomainReference(reportRef) + report = DomainReference(reportRef), + args = mutableMapOf() ) ).flatMap { Mono.empty() } } @@ -72,9 +73,10 @@ class DefaultReportDocumentChangeEventConsumer( Mono.zip( affectedReports.map { report -> createReportCalculationUseCase - .startReportCalculation( + .createReportCalculation( request = CreateReportCalculationRequest( - report = report + report = report, + args = mutableMapOf() ) ) } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/QueryStorage.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/QueryStorage.kt index 97d9968..c8ac3a0 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/QueryStorage.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/QueryStorage.kt @@ -1,9 +1,9 @@ package com.aamdigital.aambackendservice.reporting.report.core import com.aamdigital.aambackendservice.reporting.report.sqs.QueryRequest -import com.aamdigital.aambackendservice.reporting.report.sqs.QueryResult -import reactor.core.publisher.Mono +import org.springframework.core.io.buffer.DataBuffer +import reactor.core.publisher.Flux interface QueryStorage { - fun executeQuery(query: QueryRequest): Mono + fun executeQuery(query: QueryRequest): Flux } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/ReportingStorage.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/ReportingStorage.kt index b2a2c31..7df2344 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/ReportingStorage.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/core/ReportingStorage.kt @@ -1,18 +1,14 @@ package com.aamdigital.aambackendservice.reporting.report.core import com.aamdigital.aambackendservice.domain.DomainReference -import com.aamdigital.aambackendservice.reporting.domain.Report import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation -import com.aamdigital.aambackendservice.reporting.domain.ReportData +import org.springframework.core.io.buffer.DataBuffer +import org.springframework.http.HttpHeaders +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.util.* interface ReportingStorage { - fun fetchAllReports(mode: String): Mono> - fun fetchReport( - report: DomainReference - ): Mono> - fun fetchPendingCalculations(): Mono> fun fetchCalculations(reportReference: DomainReference): Mono> fun fetchCalculation( @@ -20,7 +16,9 @@ interface ReportingStorage { ): Mono> fun storeCalculation(reportCalculation: ReportCalculation): Mono - fun storeData(reportData: ReportData): Mono - fun fetchData(calculationReference: DomainReference): Mono> + + fun headData(calculationReference: DomainReference): Mono + fun fetchData(calculationReference: DomainReference): Flux + fun isCalculationOngoing(reportReference: DomainReference): Mono } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/di/ReportConfiguration.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/di/ReportConfiguration.kt index a9327be..529811a 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/di/ReportConfiguration.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/di/ReportConfiguration.kt @@ -8,6 +8,7 @@ import com.aamdigital.aambackendservice.reporting.report.core.ReportCalculationP import com.aamdigital.aambackendservice.reporting.report.core.ReportSchemaGenerator import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage import com.aamdigital.aambackendservice.reporting.reportcalculation.core.ReportCalculator +import com.aamdigital.aambackendservice.reporting.storage.DefaultReportStorage import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -17,10 +18,10 @@ class ReportConfiguration { @Bean fun defaultIdentifyAffectedReportsUseCase( - reportingStorage: ReportingStorage, + reportStorage: DefaultReportStorage, schemaGenerator: ReportSchemaGenerator, ): IdentifyAffectedReportsUseCase = - DefaultIdentifyAffectedReportsUseCase(reportingStorage, schemaGenerator) + DefaultIdentifyAffectedReportsUseCase(reportStorage, schemaGenerator) @Bean @ConditionalOnProperty( diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/dto/ControllerDtos.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/dto/ControllerDtos.kt index 13f3582..cf75693 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/dto/ControllerDtos.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/dto/ControllerDtos.kt @@ -22,6 +22,7 @@ data class ReportDoc( val title: String, val mode: String = "unknown", val aggregationDefinition: String?, + val neededArgs: List = emptyList(), val created: EditAtBy?, val updated: EditAtBy?, ) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/jobs/ReportCalculationJob.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/jobs/ReportCalculationJob.kt index 7bdfff1..2dcf172 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/jobs/ReportCalculationJob.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/jobs/ReportCalculationJob.kt @@ -18,6 +18,6 @@ class ReportCalculationJob( .doOnError { logger.error("[ReportCalculationJob] Error in job: processNextPendingCalculation()", it) } - .subscribe {} + .subscribe() } } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsQueryStorage.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsQueryStorage.kt index 97edd6b..c00e4f4 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsQueryStorage.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsQueryStorage.kt @@ -4,18 +4,18 @@ import com.aamdigital.aambackendservice.error.InvalidArgumentException import com.aamdigital.aambackendservice.reporting.report.core.QueryStorage import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.core.io.buffer.DataBuffer import org.springframework.http.MediaType import org.springframework.stereotype.Service +import org.springframework.web.reactive.function.BodyExtractors import org.springframework.web.reactive.function.BodyInserters import org.springframework.web.reactive.function.client.WebClient -import reactor.core.publisher.Mono +import reactor.core.publisher.Flux +import reactor.kotlin.core.publisher.toFlux data class QueryRequest( - val query: String -) - -data class QueryResult( - val result: List<*> + val query: String, + val args: List ) @Service @@ -25,32 +25,35 @@ class SqsQueryStorage( ) : QueryStorage { private val logger = LoggerFactory.getLogger(javaClass) - override fun executeQuery(query: QueryRequest): Mono { + override fun executeQuery(query: QueryRequest): Flux { val schemaPath = schemaService.getSchemaPath() return schemaService.updateSchema() + .toFlux() .flatMap { sqsClient.post() .uri(schemaPath) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(query)) .accept(MediaType.APPLICATION_JSON) - .exchangeToMono { response -> + .exchangeToFlux { response -> if (response.statusCode().is2xxSuccessful) { - response.bodyToMono(List::class.java) - .map { - QueryResult(result = it) - } + response.body(BodyExtractors.toDataBuffers()) } else { - response.bodyToMono(String::class.java) + response.bodyToFlux(String::class.java) .flatMap { - logger.error("[SqsQueryStorage] Invalid response from SQS: $it") - Mono.error(InvalidArgumentException(it)) + logger.error( + "[SqsQueryStorage] " + + "Invalid response (${response.statusCode()}) from SQS: $it" + ) + Flux.error(InvalidArgumentException(it)) } } } + .onErrorResume { + logger.error("[SqsQueryStorage]: ${it.localizedMessage}", it) + Flux.error(InvalidArgumentException(it.localizedMessage)) + } } - .doOnError { - logger.error("[SqsQueryStorage]: ${it.localizedMessage}", it) - } + } } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsSchemaService.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsSchemaService.kt index 1a631ac..0cb30f9 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsSchemaService.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsSchemaService.kt @@ -1,7 +1,8 @@ package com.aamdigital.aambackendservice.reporting.report.sqs -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient import com.aamdigital.aambackendservice.domain.EntityAttribute +import com.aamdigital.aambackendservice.domain.EntityAttributeType import com.aamdigital.aambackendservice.domain.EntityConfig import com.aamdigital.aambackendservice.domain.EntityType import com.fasterxml.jackson.annotation.JsonProperty @@ -31,7 +32,7 @@ data class AppConfigFile( ) data class TableFields( - val fields: Map + val fields: Map ) data class TableName( @@ -72,7 +73,7 @@ data class SqsSchema( @Service class SqsSchemaService( - private val couchDbStorage: CouchDbStorage, + private val couchDbClient: CouchDbClient, ) { companion object { @@ -86,7 +87,7 @@ class SqsSchemaService( fun updateSchema(): Mono { return Mono.zip( - couchDbStorage.getDatabaseDocument( + couchDbClient.getDatabaseDocument( database = TARGET_DATABASE, documentId = FILENAME_CONFIG_ENTITY, queryParams = LinkedMultiValueMap(), @@ -104,7 +105,7 @@ class SqsSchemaService( EntityConfig(config.rev, entities) }, - couchDbStorage.getDatabaseDocument( + couchDbClient.getDatabaseDocument( database = TARGET_DATABASE, documentId = SCHEMA_PATH, queryParams = LinkedMultiValueMap(), @@ -122,7 +123,7 @@ class SqsSchemaService( return@flatMap Mono.just(Unit) } - couchDbStorage.putDatabaseDocument( + couchDbClient.putDatabaseDocument( database = TARGET_DATABASE, documentId = SCHEMA_PATH, body = newSqsSchema, @@ -137,12 +138,15 @@ class SqsSchemaService( val tables = entityConfig.entities.map { entityType -> val attributes = entityType.attributes .filter { - it.type != "file" + it.type.type != "file" } .map { EntityAttribute( it.name, - mapConfigDataTypeToSqsDataType(it.type) + EntityAttributeType( + it.name, + type = mapConfigDataTypeToSqsDataType(it.type.type) + ) ) } .plus(getDefaultEntityAttributes()) @@ -179,12 +183,60 @@ class SqsSchemaService( } private fun getDefaultEntityAttributes(): List = listOf( - EntityAttribute("_id", "TEXT"), - EntityAttribute("_rev", "TEXT"), - EntityAttribute("created", "TEXT"), - EntityAttribute("updated", "TEXT"), - EntityAttribute("inactive", "INTEGER"), - EntityAttribute("anonymized", "INTEGER"), + EntityAttribute( + "_id", EntityAttributeType( + field = "_id", + type = "TEXT" + ) + ), + EntityAttribute( + "_rev", EntityAttributeType( + field = "_rev", + type = "TEXT" + ) + ), + EntityAttribute( + "_attachments", EntityAttributeType( + field = "_attachments", + type = "TEXT" + ) + ), + EntityAttribute( + "created_at", EntityAttributeType( + field = "created.at", + type = "DATE" + ) + ), + EntityAttribute( + "created_by", EntityAttributeType( + field = "created.by", + type = "TEXT" + ) + ), + EntityAttribute( + "updated_at", EntityAttributeType( + field = "updated.at", + type = "DATE" + ) + ), + EntityAttribute( + "updated_by", EntityAttributeType( + field = "updated.by", + type = "TEXT" + ) + ), + EntityAttribute( + "inactive", EntityAttributeType( + field = "inactive", + type = "INTEGER" + ) + ), + EntityAttribute( + "anonymized", EntityAttributeType( + field = "anonymized", + type = "INTEGER" + ) + ), ) @@ -194,7 +246,10 @@ class SqsSchemaService( attributes = config.attributes.orEmpty().map { EntityAttribute( name = it.key, - type = it.value.dataType + type = EntityAttributeType( + field = it.key, + type = it.value.dataType + ) ) } ) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/controller/ReportCalculationController.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/controller/ReportCalculationController.kt index e437a0d..65c6cc4 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/controller/ReportCalculationController.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/controller/ReportCalculationController.kt @@ -4,89 +4,155 @@ import com.aamdigital.aambackendservice.domain.DomainReference import com.aamdigital.aambackendservice.error.InternalServerException import com.aamdigital.aambackendservice.error.NotFoundException import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation -import com.aamdigital.aambackendservice.reporting.domain.ReportData import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationRequest import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationResult import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationUseCase +import com.aamdigital.aambackendservice.reporting.reportcalculation.dto.ReportCalculationDto +import com.aamdigital.aambackendservice.reporting.storage.DefaultReportStorage +import org.springframework.core.io.buffer.DataBuffer +import org.springframework.core.io.buffer.DefaultDataBufferFactory +import org.springframework.format.annotation.DateTimeFormat +import org.springframework.http.MediaType import org.springframework.validation.annotation.Validated import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController +import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import reactor.kotlin.core.publisher.toFlux +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.util.* + @RestController @RequestMapping("/v1/reporting/report-calculation") @Validated class ReportCalculationController( private val reportingStorage: ReportingStorage, - private val createReportCalculationUseCase: CreateReportCalculationUseCase + private val reportStorage: DefaultReportStorage, + private val createReportCalculationUseCase: CreateReportCalculationUseCase, ) { @PostMapping("/report/{reportId}") fun startCalculation( - @PathVariable reportId: String + @PathVariable reportId: String, + @RequestParam @DateTimeFormat(pattern = "yyyy-MM-dd") from: Date?, + @RequestParam @DateTimeFormat(pattern = "yyyy-MM-dd") to: Date?, ): Mono { - return reportingStorage.fetchReport(DomainReference(id = reportId)) - .flatMap { reportOptional -> - val report = reportOptional.orElseThrow { - NotFoundException() - } + return reportStorage.fetchReport(DomainReference(id = reportId)).flatMap { reportOptional -> + val report = reportOptional.orElseThrow { + NotFoundException() + } + + val args = mutableMapOf() - createReportCalculationUseCase.startReportCalculation( - CreateReportCalculationRequest( - report = DomainReference(report.id) - ) - ).handle { result, sink -> - when (result) { - is CreateReportCalculationResult.Failure -> { - sink.error(InternalServerException()) - } - - is CreateReportCalculationResult.Success -> sink.next(result.calculation) + if (from != null) { + args["from"] = from.toInstant().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_DATE_TIME) + } + + if (to != null) { + args["to"] = to.toInstant().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_DATE_TIME) + } + + createReportCalculationUseCase.createReportCalculation( + CreateReportCalculationRequest( + report = DomainReference(report.id), args = args + ) + ).handle { result, sink -> + when (result) { + is CreateReportCalculationResult.Failure -> { + sink.error(InternalServerException()) } + + is CreateReportCalculationResult.Success -> sink.next(result.calculation) } } + } } @GetMapping("/report/{reportId}") fun fetchReportCalculations( @PathVariable reportId: String - ): Mono> { - return reportingStorage.fetchCalculations(DomainReference(id = reportId)) + ): Mono> { + return reportingStorage.fetchCalculations(DomainReference(id = reportId)).map { calculations -> + calculations.map { toDto(it) } + } } @GetMapping("/{calculationId}") fun fetchReportCalculation( @PathVariable calculationId: String - ): Mono { - return reportingStorage.fetchCalculation(DomainReference(id = calculationId)) - .map { calculationOptional -> - val calculation = calculationOptional.orElseThrow { - NotFoundException() - } + ): Mono { + return reportingStorage.fetchCalculation(DomainReference(id = calculationId)).map { calculationOptional -> + val calculation = calculationOptional.orElseThrow { + NotFoundException() + } - // TODO Auth check + // TODO Auth check (https://github.com/Aam-Digital/aam-services/issues/10) - calculation - } + toDto(calculation) + } } - @GetMapping("/{calculationId}/data") + @GetMapping("/{calculationId}/data", produces = [MediaType.APPLICATION_JSON_VALUE]) fun fetchReportCalculationData( @PathVariable calculationId: String - ): Mono { - return reportingStorage.fetchData(DomainReference(id = calculationId)) - .map { calculationOptional -> - val calculation = calculationOptional.orElseThrow { - NotFoundException() + ): Flux { + // TODO Auth check (https://github.com/Aam-Digital/aam-services/issues/10) + + return reportingStorage.headData(DomainReference(calculationId)) + .toFlux() + .flatMap { + if (it.eTag.isNullOrBlank()) { + return@flatMap Flux.error { NotFoundException("No data available") } } - // TODO Auth check + val fileContent = reportingStorage + .fetchData(DomainReference(id = calculationId)) + + val prefix = """ + { + "id": "${calculationId}_data.json", + "report": "removed", + "calculation": "$calculationId", + "data": + """.trimIndent().toByteArray() + val prefixBuffer = DefaultDataBufferFactory().allocateBuffer(prefix.size) + prefixBuffer.write(prefix) - calculation + val suffix = """ + } + """.trimIndent().toByteArray() + val suffixBuffer = DefaultDataBufferFactory().allocateBuffer(suffix.size) + suffixBuffer.write(suffix) + + return@flatMap Flux.concat( + Flux.just(prefixBuffer), + fileContent, + Flux.just(suffixBuffer), + ) } } + @GetMapping("/{calculationId}/data-stream", produces = [MediaType.APPLICATION_OCTET_STREAM_VALUE]) + fun fetchReportCalculationDataStream( + @PathVariable calculationId: String + ): Flux { + // TODO Auth check (https://github.com/Aam-Digital/aam-services/issues/10) + return reportingStorage.fetchData(DomainReference(id = calculationId)) + } + + private fun toDto(it: ReportCalculation): ReportCalculationDto = ReportCalculationDto( + id = it.id, + report = it.report, + status = it.status, + startDate = it.calculationStarted, + endDate = it.calculationCompleted, + args = it.args, + attachments = it.attachments, + ) } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/CreateReportCalculationUseCase.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/CreateReportCalculationUseCase.kt index 9168395..72f6a27 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/CreateReportCalculationUseCase.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/CreateReportCalculationUseCase.kt @@ -7,6 +7,7 @@ import reactor.core.publisher.Mono data class CreateReportCalculationRequest( val report: DomainReference, + val args: MutableMap, ) @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) @@ -31,5 +32,5 @@ sealed class CreateReportCalculationResult { } interface CreateReportCalculationUseCase { - fun startReportCalculation(request: CreateReportCalculationRequest): Mono + fun createReportCalculation(request: CreateReportCalculationRequest): Mono } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultCreateReportCalculationUseCase.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultCreateReportCalculationUseCase.kt index e5a0db1..c9bc301 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultCreateReportCalculationUseCase.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultCreateReportCalculationUseCase.kt @@ -12,11 +12,13 @@ import java.util.* class DefaultCreateReportCalculationUseCase( private val reportingStorage: ReportingStorage, ) : CreateReportCalculationUseCase { - override fun startReportCalculation(request: CreateReportCalculationRequest): Mono { + + override fun createReportCalculation(request: CreateReportCalculationRequest): Mono { val calculation = ReportCalculation( id = "ReportCalculation:${UUID.randomUUID()}", report = request.report, status = ReportCalculationStatus.PENDING, + args = request.args ) return reportingStorage.storeCalculation(calculation) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculationChangeUseCase.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculationChangeUseCase.kt index 4eb28b5..51758ef 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculationChangeUseCase.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculationChangeUseCase.kt @@ -33,10 +33,15 @@ class DefaultReportCalculationChangeUseCase( .map { calculations -> calculations .filter { it.id != currentReportCalculation.id } - .sortedBy { it.endDate } + .sortedBy { it.calculationCompleted } } .flatMap { - if (it.isEmpty() || it.last().outcome != currentReportCalculation.outcome) { + val existingDigest = it.last().attachments["data.json"]?.digest + val currentDigest = currentReportCalculation.attachments["data.json"]?.digest + + if (it.isEmpty() + || existingDigest != currentDigest + ) { notificationService.sendNotifications( report = currentReportCalculation.report, reportCalculation = DomainReference(currentReportCalculation.id) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculator.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculator.kt index 6c58651..522bc6a 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculator.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/DefaultReportCalculator.kt @@ -1,25 +1,30 @@ package com.aamdigital.aambackendservice.reporting.reportcalculation.core -import com.aamdigital.aambackendservice.domain.DomainReference +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient import com.aamdigital.aambackendservice.error.InvalidArgumentException import com.aamdigital.aambackendservice.error.NotFoundException import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation -import com.aamdigital.aambackendservice.reporting.domain.ReportData import com.aamdigital.aambackendservice.reporting.report.core.QueryStorage -import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage import com.aamdigital.aambackendservice.reporting.report.sqs.QueryRequest +import com.aamdigital.aambackendservice.reporting.storage.DefaultReportStorage import org.springframework.stereotype.Service import reactor.core.publisher.Mono -import java.util.* import kotlin.jvm.optionals.getOrDefault @Service class DefaultReportCalculator( - private val reportingStorage: ReportingStorage, + private val reportStorage: DefaultReportStorage, private val queryStorage: QueryStorage, + private val couchDbClient: CouchDbClient, ) : ReportCalculator { - override fun calculate(reportCalculation: ReportCalculation): Mono { - return reportingStorage.fetchReport(reportCalculation.report) + + companion object { + const val DEFAULT_FROM_DATE = "0000-01-01T00:00:00.000Z" + const val DEFAULT_TO_DATE = "9999-12-31T23:59:59.999Z" + } + + override fun calculate(reportCalculation: ReportCalculation): Mono { + return reportStorage.fetchReport(reportCalculation.report) .flatMap { reportOptional -> val report = reportOptional.getOrDefault(null) ?: return@flatMap Mono.error(NotFoundException()) @@ -28,17 +33,44 @@ class DefaultReportCalculator( return@flatMap Mono.error(InvalidArgumentException()) } - queryStorage.executeQuery( - query = QueryRequest(query = report.query) + setToDateToLastMinuteOfDay(reportCalculation.args) + + val queryResult = queryStorage.executeQuery( + query = QueryRequest( + query = report.query, + args = getReportCalculationArgs(report.neededArgs, reportCalculation.args) + ) ) - .map { queryResult -> - ReportData( - id = "ReportData:${UUID.randomUUID()}", - report = reportCalculation.report, - calculation = DomainReference(reportCalculation.id), - data = queryResult.result - ) - } + + couchDbClient.putAttachment( + database = "report-calculation", + documentId = reportCalculation.id, + attachmentId = "data.json", + file = queryResult, + ).map { + reportCalculation + } } } + + private fun setToDateToLastMinuteOfDay(args: MutableMap) { + val toDateString = args["to"] ?: return + args["to"] = toDateString.substring(IntRange(0, 9)) + "T23:59:59.999Z" + } + + private fun getReportCalculationArgs(neededArgs: List, givenArgs: Map): List = + neededArgs + .map { + givenArgs[it] + ?: getDefaultValue(it) + ?: throw NotFoundException( + "Argument $it is missing. All report args are needed for a successful ReportCalculation." + ) + } + + private fun getDefaultValue(arg: String): String? = when (arg) { + "from" -> DEFAULT_FROM_DATE + "to" -> DEFAULT_TO_DATE + else -> null + } } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/ReportCalculator.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/ReportCalculator.kt index 641cf2b..ca71825 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/ReportCalculator.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/core/ReportCalculator.kt @@ -1,9 +1,8 @@ package com.aamdigital.aambackendservice.reporting.reportcalculation.core import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation -import com.aamdigital.aambackendservice.reporting.domain.ReportData import reactor.core.publisher.Mono interface ReportCalculator { - fun calculate(reportCalculation: ReportCalculation): Mono + fun calculate(reportCalculation: ReportCalculation): Mono } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/dto/ControllerDtos.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/dto/ControllerDtos.kt new file mode 100644 index 0000000..442c650 --- /dev/null +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/dto/ControllerDtos.kt @@ -0,0 +1,29 @@ +package com.aamdigital.aambackendservice.reporting.reportcalculation.dto + +import com.aamdigital.aambackendservice.couchdb.dto.AttachmentMetaData +import com.aamdigital.aambackendservice.domain.DomainReference +import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationStatus + +/** + * This is the interface shared to external users of the API endpoints. + */ +data class ReportCalculationDto( + val id: String, + val report: DomainReference, + var status: ReportCalculationStatus, + var startDate: String? = null, + var endDate: String? = null, + var args: Map, + var attachments: Map = emptyMap(), +) + +/** + * used by /data endpoint in ReportCalculationController, but response is build dynamically + * Just kept for backwarts compatibility + */ +data class ReportDataDto( + val id: String, + val report: DomainReference, + val calculation: DomainReference, + val data: String +) diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportStorage.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportStorage.kt new file mode 100644 index 0000000..df15e3f --- /dev/null +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportStorage.kt @@ -0,0 +1,77 @@ +package com.aamdigital.aambackendservice.reporting.storage + +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient +import com.aamdigital.aambackendservice.couchdb.core.getQueryParamsAllDocs +import com.aamdigital.aambackendservice.domain.DomainReference +import com.aamdigital.aambackendservice.reporting.domain.Report +import com.aamdigital.aambackendservice.reporting.domain.ReportSchema +import com.aamdigital.aambackendservice.reporting.report.core.ReportSchemaGenerator +import com.aamdigital.aambackendservice.reporting.report.dto.FetchReportsResponse +import com.aamdigital.aambackendservice.reporting.report.dto.ReportDoc +import org.springframework.stereotype.Service +import org.springframework.util.LinkedMultiValueMap +import reactor.core.publisher.Mono +import java.util.* + +@Service +// todo add ReportStorage interface +class DefaultReportStorage( + private val couchDbClient: CouchDbClient, + private val reportSchemaGenerator: ReportSchemaGenerator, +) { + companion object { + private const val REPORT_DATABASE = "app" + } + + fun fetchAllReports(mode: String): Mono> { + return couchDbClient.getDatabaseDocument( + database = REPORT_DATABASE, + documentId = "_all_docs", + getQueryParamsAllDocs("ReportConfig"), + FetchReportsResponse::class + ) + .map { response -> + if (response.rows.isEmpty()) { + return@map emptyList() + } + + response.rows.filter { + it.doc.mode == mode + }.map { + Report( + id = it.id, + name = it.doc.title, + mode = it.doc.mode, + query = it.doc.aggregationDefinition ?: "", + schema = ReportSchema( + fields = reportSchemaGenerator.getTableNamesByQuery(it.doc.aggregationDefinition ?: "") + ), + neededArgs = it.doc.neededArgs + ) + } + } + } + + fun fetchReport(report: DomainReference): Mono> { + return couchDbClient.getDatabaseDocument( + database = REPORT_DATABASE, + documentId = report.id, + queryParams = LinkedMultiValueMap(), + kClass = ReportDoc::class + ).map { reportDoc -> + Optional.of( + Report( + id = reportDoc.id, + name = reportDoc.title, + query = reportDoc.aggregationDefinition ?: "", + mode = reportDoc.mode, + schema = ReportSchema( + fields = reportSchemaGenerator.getTableNamesByQuery(reportDoc.aggregationDefinition ?: "") + ), + neededArgs = reportDoc.neededArgs + ) + ) + } + .onErrorReturn(Optional.empty()) + } +} diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportingStorage.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportingStorage.kt index 2b63235..965861c 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportingStorage.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportingStorage.kt @@ -3,16 +3,14 @@ package com.aamdigital.aambackendservice.reporting.storage import com.aamdigital.aambackendservice.couchdb.dto.CouchDbChange import com.aamdigital.aambackendservice.domain.DomainReference import com.aamdigital.aambackendservice.error.NotFoundException -import com.aamdigital.aambackendservice.reporting.domain.Report import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationStatus -import com.aamdigital.aambackendservice.reporting.domain.ReportData -import com.aamdigital.aambackendservice.reporting.domain.ReportSchema -import com.aamdigital.aambackendservice.reporting.report.core.ReportSchemaGenerator import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage import com.fasterxml.jackson.annotation.JsonProperty +import org.springframework.core.io.buffer.DataBuffer +import org.springframework.http.HttpHeaders import org.springframework.stereotype.Service -import org.springframework.util.LinkedMultiValueMap +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.util.* @@ -31,54 +29,10 @@ data class FetchReportCalculationsResponse( ) @Service +@Deprecated("use sub storages directly") class DefaultReportingStorage( - private val reportRepository: ReportRepository, private val reportCalculationRepository: ReportCalculationRepository, - private val reportSchemaGenerator: ReportSchemaGenerator, ) : ReportingStorage { - override fun fetchAllReports(mode: String): Mono> { - return reportRepository.fetchReports() - .map { response -> - if (response.rows.isEmpty()) { - return@map emptyList() - } - - response.rows.filter { - it.doc.mode == mode - }.map { - Report( - id = it.id, - name = it.doc.title, - mode = it.doc.mode, - query = it.doc.aggregationDefinition ?: "", - schema = ReportSchema( - fields = reportSchemaGenerator.getTableNamesByQuery(it.doc.aggregationDefinition ?: "") - ) - ) - } - } - } - - override fun fetchReport(report: DomainReference): Mono> { - return reportRepository.fetchReport( - documentId = report.id, - queryParams = LinkedMultiValueMap(), - ).map { reportDoc -> - Optional.of( - Report( - id = reportDoc.id, - name = reportDoc.title, - query = reportDoc.aggregationDefinition ?: "", - mode = reportDoc.mode, - schema = ReportSchema( - fields = reportSchemaGenerator.getTableNamesByQuery(reportDoc.aggregationDefinition ?: "") - ) - ) - ) - } - .onErrorReturn(Optional.empty()) - } - override fun fetchPendingCalculations(): Mono> { return reportCalculationRepository.fetchCalculations() .map { response -> @@ -117,12 +71,12 @@ class DefaultReportingStorage( } } - override fun storeData(reportData: ReportData): Mono { - return reportCalculationRepository.storeData(reportData) + override fun fetchData(calculationReference: DomainReference): Flux { + return reportCalculationRepository.fetchData(calculationReference) } - override fun fetchData(calculationReference: DomainReference): Mono> { - return reportCalculationRepository.fetchData(calculationReference) + override fun headData(calculationReference: DomainReference): Mono { + return reportCalculationRepository.headData(calculationReference) } override fun isCalculationOngoing(reportReference: DomainReference): Mono { @@ -143,8 +97,10 @@ class DefaultReportingStorage( id = entity.doc.id, report = entity.doc.report, status = entity.doc.status, - startDate = entity.doc.startDate, - endDate = entity.doc.endDate, - outcome = entity.doc.outcome + calculationStarted = entity.doc.calculationStarted, + calculationCompleted = entity.doc.calculationCompleted, + args = entity.doc.args, + attachments = entity.doc.attachments + ) } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportCalculationRepository.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportCalculationRepository.kt index c9422df..1839522 100644 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportCalculationRepository.kt +++ b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportCalculationRepository.kt @@ -1,21 +1,21 @@ package com.aamdigital.aambackendservice.reporting.storage -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage +import com.aamdigital.aambackendservice.couchdb.core.CouchDbClient import com.aamdigital.aambackendservice.couchdb.core.getQueryParamsAllDocs import com.aamdigital.aambackendservice.couchdb.dto.DocSuccess import com.aamdigital.aambackendservice.domain.DomainReference -import com.aamdigital.aambackendservice.error.NotFoundException import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation -import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationOutcome -import com.aamdigital.aambackendservice.reporting.domain.ReportData +import org.springframework.core.io.buffer.DataBuffer +import org.springframework.http.HttpHeaders import org.springframework.stereotype.Service import org.springframework.util.LinkedMultiValueMap +import reactor.core.publisher.Flux import reactor.core.publisher.Mono import java.util.* @Service class ReportCalculationRepository( - private val couchDbStorage: CouchDbStorage, + private val couchDbClient: CouchDbClient, ) { companion object { private const val REPORT_CALCULATION_DATABASE = "report-calculation" @@ -24,7 +24,7 @@ class ReportCalculationRepository( fun storeCalculation( reportCalculation: ReportCalculation, ): Mono { - return couchDbStorage.putDatabaseDocument( + return couchDbClient.putDatabaseDocument( database = REPORT_CALCULATION_DATABASE, documentId = reportCalculation.id, body = reportCalculation, @@ -32,7 +32,7 @@ class ReportCalculationRepository( } fun fetchCalculations(): Mono { - return couchDbStorage.getDatabaseDocument( + return couchDbClient.getDatabaseDocument( database = REPORT_CALCULATION_DATABASE, documentId = "_all_docs", getQueryParamsAllDocs("ReportCalculation"), @@ -41,7 +41,7 @@ class ReportCalculationRepository( } fun fetchCalculation(calculationReference: DomainReference): Mono> { - return couchDbStorage.getDatabaseDocument( + return couchDbClient.getDatabaseDocument( database = REPORT_CALCULATION_DATABASE, documentId = calculationReference.id, queryParams = LinkedMultiValueMap(), @@ -52,62 +52,19 @@ class ReportCalculationRepository( .onErrorReturn(Optional.empty()) } - fun storeData(data: ReportData): Mono { - return couchDbStorage.putDatabaseDocument( - database = REPORT_CALCULATION_DATABASE, - documentId = data.id, - body = data, + fun headData(calculationReference: DomainReference): Mono { + return couchDbClient.headAttachment( + database = "report-calculation", + documentId = calculationReference.id, + attachmentId = "data.json", ) - .flatMap { fetchCalculation(data.calculation) } - .flatMap { - val calculation = it.orElseThrow { - NotFoundException() - } - - calculation.outcome = ReportCalculationOutcome.Success( - resultHash = data.getDataHash() - ) - - couchDbStorage.putDatabaseDocument( - database = REPORT_CALCULATION_DATABASE, - documentId = calculation.id, - body = calculation - ) - }.map { data } } - fun fetchData(calculationReference: DomainReference): Mono> { - return fetchCalculation(calculationReference) - .map { - val calculation = it.orElseThrow { - NotFoundException() - } - calculation.id - } - .flatMap { calculationId -> - couchDbStorage.find( - database = "report-calculation", - body = mapOf( - Pair( - "selector", mapOf( - Pair( - "calculation.id", mapOf( - Pair("\$eq", calculationId) - ) - ) - ) - ) - ), - queryParams = LinkedMultiValueMap(), - kClass = ReportData::class - ) - } - .map { - if (it.docs.size == 1) { - Optional.of(it.docs.first()) - } else { - Optional.empty() - } - } + fun fetchData(calculationReference: DomainReference): Flux { + return couchDbClient.getAttachment( + database = "report-calculation", + documentId = calculationReference.id, + attachmentId = "data.json", + ) } } diff --git a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportRepository.kt b/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportRepository.kt deleted file mode 100644 index 911186e..0000000 --- a/application/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/reporting/storage/ReportRepository.kt +++ /dev/null @@ -1,36 +0,0 @@ -package com.aamdigital.aambackendservice.reporting.storage - -import com.aamdigital.aambackendservice.couchdb.core.CouchDbStorage -import com.aamdigital.aambackendservice.couchdb.core.getQueryParamsAllDocs -import com.aamdigital.aambackendservice.reporting.report.dto.FetchReportsResponse -import com.aamdigital.aambackendservice.reporting.report.dto.ReportDoc -import org.springframework.stereotype.Service -import org.springframework.util.LinkedMultiValueMap -import reactor.core.publisher.Mono - -@Service -class ReportRepository( - private val couchDbStorage: CouchDbStorage, -) { - companion object { - private const val REPORT_DATABASE = "app" - } - - fun fetchReports(): Mono { - return couchDbStorage.getDatabaseDocument( - database = REPORT_DATABASE, - documentId = "_all_docs", - getQueryParamsAllDocs("ReportConfig"), - FetchReportsResponse::class - ) - } - - fun fetchReport(documentId: String, queryParams: LinkedMultiValueMap): Mono { - return couchDbStorage.getDatabaseDocument( - database = REPORT_DATABASE, - documentId = documentId, - queryParams, - ReportDoc::class - ) - } -} diff --git a/application/aam-backend-service/src/main/resources/application.yaml b/application/aam-backend-service/src/main/resources/application.yaml index 630d407..d865b84 100644 --- a/application/aam-backend-service/src/main/resources/application.yaml +++ b/application/aam-backend-service/src/main/resources/application.yaml @@ -26,7 +26,6 @@ management: sqs-client-configuration: max-in-memory-size-in-mega-bytes: 16 - couch-db-client-configuration: max-in-memory-size-in-mega-bytes: 16 @@ -74,7 +73,7 @@ sqs-client-configuration: basic-auth-password: docker database-change-detection: - enabled: false + enabled: true report-calculation-processor: enabled: true diff --git a/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/common/CouchDbTestingService.kt b/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/common/CouchDbTestingService.kt index 25b3566..31792a1 100644 --- a/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/common/CouchDbTestingService.kt +++ b/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/common/CouchDbTestingService.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode import org.slf4j.LoggerFactory import org.springframework.http.HttpEntity import org.springframework.http.HttpMethod +import org.springframework.util.LinkedMultiValueMap import org.springframework.web.client.RestTemplate class CouchDbTestingService( @@ -57,6 +58,30 @@ class CouchDbTestingService( logger.info("[CouchDbSetup] create Document: $database, $documentName, ${response.statusCode}") } + fun addAttachment(database: String, documentName: String, attachmentName: String, documentContent: String) { + val responseDocument = restTemplate + .headForHeaders( + "/$database/$documentName", + ) + + val etag = responseDocument.eTag?.replace("\"", "") + + val headers = LinkedMultiValueMap() + headers.set("If-Match", etag) + + val response = restTemplate + .exchange( + "/$database/$documentName/$attachmentName", + HttpMethod.PUT, + HttpEntity(documentContent, headers), + ObjectNode::class.java, + ) + logger.info( + "[CouchDbSetup] create attachment:" + + " $database, $documentName, $attachmentName, ${response.statusCode}" + ) + } + private fun deleteDatabase(database: String) { val response = restTemplate .exchange( diff --git a/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/e2e/CucumberIntegrationTest.kt b/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/e2e/CucumberIntegrationTest.kt index e6ab4c4..a238d86 100644 --- a/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/e2e/CucumberIntegrationTest.kt +++ b/application/aam-backend-service/src/test/kotlin/com/aamdigital/aambackendservice/e2e/CucumberIntegrationTest.kt @@ -34,6 +34,16 @@ class CucumberIntegrationTest : SpringIntegrationTest() { couchDbTestingService.createDatabase(name) } + @Given("attachment {} added to document {} in {}") + fun `store attachment in document`(attachment: String, document: String, database: String) { + couchDbTestingService.addAttachment( + database = database, + documentName = document, + attachmentName = "data.json", // fixed in business logic for now + documentContent = File("src/test/resources/database/documents/$attachment.json").readText() + ) + } + @Given("document {} is stored in database {}") fun `store document in database`(document: String, database: String) { couchDbTestingService.createDocument( @@ -42,7 +52,7 @@ class CucumberIntegrationTest : SpringIntegrationTest() { documentContent = File("src/test/resources/database/documents/$document.json").readText() ) } - + @When("the client calls GET {word}") @Throws(Throwable::class) fun `the client issues GET endpoint`(endpoint: String) { diff --git a/application/aam-backend-service/src/test/resources/application-e2e.yaml b/application/aam-backend-service/src/test/resources/application-e2e.yaml index bbeedca..7dc2826 100644 --- a/application/aam-backend-service/src/test/resources/application-e2e.yaml +++ b/application/aam-backend-service/src/test/resources/application-e2e.yaml @@ -34,11 +34,13 @@ management: - health couch-db-client-configuration: + max-in-memory-size-in-mega-bytes: 16 base-path: http://localhost:5984 basic-auth-username: admin basic-auth-password: docker sqs-client-configuration: + max-in-memory-size-in-mega-bytes: 16 base-path: http://localhost:4984 basic-auth-username: admin basic-auth-password: docker diff --git a/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-calculation-data.feature b/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-calculation-data.feature index 7e162ec..570cd74 100644 --- a/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-calculation-data.feature +++ b/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-calculation-data.feature @@ -5,13 +5,12 @@ Feature: the report calculation data endpoint persist to database Given database report-calculation is created Given document ReportConfig:1 is stored in database app Given document ReportCalculation:1 is stored in database report-calculation - Given document ReportData:1 is stored in database report-calculation + Given attachment ReportData:1 added to document ReportCalculation:1 in report-calculation Given signed in as client dummy-client with secret client-secret in realm dummy-realm When the client calls GET /v1/reporting/report-calculation/ReportCalculation:1/data Then the client receives an json object Then the client receives status code of 200 - Then the client receives value 22cfbb91fcbff5a2e0755cc26567a81078898dfe939d07c9745149f45863ca31 for property dataHash - Then the client receives value ReportData:1 for property _id + Then the client receives value ReportCalculation:1_data.json for property id # ReportCalculation not available Scenario: client makes call to GET /reporting/report-calculation/ReportCalculation:42/data and receives NotFound @@ -22,7 +21,7 @@ Feature: the report calculation data endpoint persist to database Then the client receives an json object Then the client receives status code of 404 - # ReportData not available + # Data is not available yet Scenario: client makes call to GET /reporting/report-calculation/ReportCalculation:42/data and receives NotFound 2 Given database app is created Given database report-calculation is created diff --git a/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-report-calculation.feature b/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-report-calculation.feature index 105ce3d..72f47d6 100644 --- a/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-report-calculation.feature +++ b/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/fetch-report-calculation.feature @@ -39,7 +39,7 @@ Feature: the report calculation endpoint persist to database Then the client receives an json object Then the client receives status code of 200 Then the client receives value FINISHED_SUCCESS for property status - Then the client receives value ReportCalculation:1 for property _id + Then the client receives value ReportCalculation:1 for property id Scenario: client makes call to GET /reporting/report-calculation/ReportCalculation:42 and receives NotFound Given database app is created diff --git a/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/start-report-calculation.feature b/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/start-report-calculation.feature index e9122a5..01efab4 100644 --- a/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/start-report-calculation.feature +++ b/application/aam-backend-service/src/test/resources/cucumber/features/reporting/reportcalculation/start-report-calculation.feature @@ -57,3 +57,21 @@ Feature: the report calculation endpoint persist to database When the client calls GET /v1/reporting/report-calculation/ReportCalculation:2/data Then the client receives an json object Then the client receives status code of 200 + + Scenario: ReportCalculation with arguments is processed within 30 seconds + Given document ReportConfig:3 is stored in database app + Given document Config:CONFIG_ENTITY is stored in database app + Given document ReportCalculation:3 is stored in database report-calculation + Given signed in as client dummy-client with secret client-secret in realm dummy-realm + When the client calls GET /v1/reporting/report-calculation/ReportCalculation:3 + Then the client receives an json object + Then the client receives status code of 200 + Then the client receives value PENDING for property status + Then the client waits for 15000 milliseconds + When the client calls GET /v1/reporting/report-calculation/ReportCalculation:3 + Then the client receives an json object + Then the client receives status code of 200 + Then the client receives value FINISHED_SUCCESS for property status + When the client calls GET /v1/reporting/report-calculation/ReportCalculation:3/data + Then the client receives an json object + Then the client receives status code of 200 diff --git a/application/aam-backend-service/src/test/resources/database/documents/Config:CONFIG_ENTITY.json b/application/aam-backend-service/src/test/resources/database/documents/Config:CONFIG_ENTITY.json index 3f636e6..9327873 100644 --- a/application/aam-backend-service/src/test/resources/database/documents/Config:CONFIG_ENTITY.json +++ b/application/aam-backend-service/src/test/resources/database/documents/Config:CONFIG_ENTITY.json @@ -83,6 +83,10 @@ "dataType": "string", "label": "School Timing" }, + "date": { + "dataType": "data", + "label": "Create Date" + }, "remarks": { "dataType": "string", "label": "Remarks" diff --git a/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:1.json b/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:1.json index 5808fa1..d22a3a0 100644 --- a/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:1.json +++ b/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:1.json @@ -3,10 +3,11 @@ "report": { "id": "ReportConfig:1" }, + "args": { + "from": "2024-01-01T00:00:00Z", + "to": "2024-04-30T00:00:00Z" + }, "status": "FINISHED_SUCCESS", "startDate": "2024-03-20T11:42:08.571", - "endDate": "2024-03-20T11:42:08.7", - "outcome": { - "result_hash": "000" - } + "endDate": "2024-03-20T11:42:08.7" } diff --git a/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:2.json b/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:2.json index 6098676..c38ee61 100644 --- a/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:2.json +++ b/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:2.json @@ -3,8 +3,11 @@ "report": { "id": "ReportConfig:1" }, + "args": { + "from": "2024-01-01T00:00:00Z", + "to": "2024-04-30T00:00:00Z" + }, "status": "PENDING", "startDate": null, - "endDate": null, - "outcome": null + "endDate": null } diff --git a/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:3.json b/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:3.json new file mode 100644 index 0000000..f4cb5b9 --- /dev/null +++ b/application/aam-backend-service/src/test/resources/database/documents/ReportCalculation:3.json @@ -0,0 +1,13 @@ +{ + "_id": "ReportCalculation:3", + "report": { + "id": "ReportConfig:3" + }, + "args": { + "from": "2024-01-01T00:00:00Z", + "to": "2024-04-30T00:00:00Z" + }, + "status": "PENDING", + "startDate": null, + "endDate": null +} diff --git a/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:1.json b/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:1.json index b685300..6a58c4e 100644 --- a/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:1.json +++ b/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:1.json @@ -2,5 +2,6 @@ "_id": "ReportConfig:1", "title": "Test Report 1", "mode": "sql", + "neededArgs": [], "aggregationDefinition": "SELECT s.name as name, s.privateSchool as privateSchool FROM School as s" } diff --git a/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:2.json b/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:2.json index 88f2b9e..2d02547 100644 --- a/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:2.json +++ b/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:2.json @@ -2,5 +2,6 @@ "_id": "ReportConfig:2", "title": "Test Report 2", "mode": "sql", + "neededArgs": [], "aggregationDefinition": "SELECT s.name as name, s.privateSchool as privateSchool FROM School as s" } diff --git a/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:3.json b/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:3.json new file mode 100644 index 0000000..baced96 --- /dev/null +++ b/application/aam-backend-service/src/test/resources/database/documents/ReportConfig:3.json @@ -0,0 +1,10 @@ +{ + "_id": "ReportConfig:3", + "title": "Test Report 3", + "mode": "sql", + "neededArgs": [ + "from", + "to" + ], + "aggregationDefinition": "SELECT s.name as name, s.privateSchool as privateSchool FROM School as s WHERE s.date BETWEEN ? AND ?" +} diff --git a/application/aam-backend-service/src/test/resources/database/documents/ReportData:1.json b/application/aam-backend-service/src/test/resources/database/documents/ReportData:1.json index 050bf77..7a00544 100644 --- a/application/aam-backend-service/src/test/resources/database/documents/ReportData:1.json +++ b/application/aam-backend-service/src/test/resources/database/documents/ReportData:1.json @@ -1,16 +1,6 @@ -{ - "_id": "ReportData:1", - "report": { - "id": "ReportConfig:1" - }, - "calculation": { - "id": "ReportCalculation:1" - }, - "data": [ - { - "foo": "bar", - "bar": "foo" - } - ], - "dataHash": "000" -} +[ + { + "foo": "bar", + "bar": "foo" + } +] diff --git a/docs/api-specs/reporting-api-v1.yaml b/docs/api-specs/reporting-api-v1.yaml index 5173448..e1638bc 100644 --- a/docs/api-specs/reporting-api-v1.yaml +++ b/docs/api-specs/reporting-api-v1.yaml @@ -4,7 +4,7 @@ info: description: |- API to manage reports that provide data calculated based on any entities of the Aam Digital system and offer notifications when data of such reports changes. - version: 1.0.0-alpha.2 + version: 1.0.0-alpha.3 servers: - url: https://{customerId}.aam-digital.net/api/v1/reporting description: Developer Instance for testing @@ -231,6 +231,39 @@ paths: schema: $ref: '#/components/schemas/Error' + /report-calculation/{calculationId}/data-stream: + get: + security: + - development: + - reporting_read + tags: + - reporting + summary: Fetch just the report data for a specific calculation as stream + parameters: + - in: path + name: calculationId + schema: + type: string + required: true + responses: + 200: + description: The actual data that has been calculated by the calculation run. This matches the schema defined in the /report/id + content: + application/octet-stream: + example: [ { "foo": { "bar": "do" } } ] + 404: + description: report data is not available yet (when either the calculation id is not valid or the calculation is still running) + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + 401: + description: If the access token does not grant permission to this Report + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + # notification /webhook: @@ -456,6 +489,19 @@ components: type: string example: date nullable: true + params: + type: object + properties: + from: + type: string + description: optional start date for the time period of the data included in the report + format: date + nullable: true + to: + type: string + description: optional end date filtering data included in the report. If no date is given here, all data (possibly filtered by the "from" date) is included. The field considered for date filtering are defined in each report's query specifically. + format: date + nullable: true status: type: string description: Current status of the run