feat: date range support for sql reports

Aam-Digital · Jun 7, 2024 · d4d821e · d4d821e
1 parent 1ed881f
commit d4d821e
Show file tree

Hide file tree

Showing 16 changed files with 136 additions and 20 deletions.
diff --git a/...ation/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/Application.kt b/...ation/aam-backend-service/src/main/kotlin/com/aamdigital/aambackendservice/Application.kt
@@ -6,13 +6,16 @@ import org.springframework.boot.runApplication
 import org.springframework.scheduling.annotation.EnableScheduling
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RestController
+import java.util.*
 
 @SpringBootApplication
 @ConfigurationPropertiesScan
 @EnableScheduling
 class Application
 
 fun main(args: Array<String>) {
+    TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
+
     runApplication<Application>(*args)
 }
 

diff --git a/...rvice/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/di/CouchDbConfiguration.kt b/...rvice/src/main/kotlin/com/aamdigital/aambackendservice/couchdb/di/CouchDbConfiguration.kt
@@ -21,13 +21,18 @@ class CouchDbConfiguration {
     ): CouchDbStorage = CouchDbClient(webClient, objectMapper)
 
     @Bean(name = ["couch-db-client"])
-    fun couchDbWebClient(couchDbClientConfiguration: CouchDbClientConfiguration): WebClient {
+    fun couchDbWebClient(configuration: CouchDbClientConfiguration): WebClient {
         val clientBuilder =
-            WebClient.builder().baseUrl(couchDbClientConfiguration.basePath)
+            WebClient.builder()
+                .codecs {
+                    it.defaultCodecs()
+                        .maxInMemorySize(configuration.maxInMemorySizeInMegaBytes * 1024 * 1024)
+                }
+                .baseUrl(configuration.basePath)
                 .defaultHeaders {
                     it.setBasicAuth(
-                        couchDbClientConfiguration.basicAuthUsername,
-                        couchDbClientConfiguration.basicAuthPassword,
+                        configuration.basicAuthUsername,
+                        configuration.basicAuthPassword,
                     )
                 }
         return clientBuilder.clientConnector(ReactorClientHttpConnector(HttpClient.create())).build()
@@ -39,4 +44,5 @@ class CouchDbClientConfiguration(
     val basePath: String,
     val basicAuthUsername: String,
     val basicAuthPassword: String,
+    val maxInMemorySizeInMegaBytes: Int = 16,
 )
diff --git a/...ce/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportCalculation.kt b/...ce/src/main/kotlin/com/aamdigital/aambackendservice/reporting/domain/ReportCalculation.kt
@@ -22,13 +22,19 @@ sealed class ReportCalculationOutcome {
     ) : ReportCalculationOutcome()
 }
 
+data class ReportCalculationParams(
+    val from: String?,
+    val to: String?,
+)
+
 data class ReportCalculation(
     @JsonProperty("_id")
     val id: String,
     val report: DomainReference,
     var status: ReportCalculationStatus,
     var startDate: String? = null,
     var endDate: String? = null,
+    var params: ReportCalculationParams?,
     var outcome: ReportCalculationOutcome? = null,
 ) {
     fun setStatus(status: ReportCalculationStatus): ReportCalculation {
@@ -50,6 +56,11 @@ data class ReportCalculation(
         this.outcome = outcome
         return this
     }
+
+    fun setParams(params: ReportCalculationParams): ReportCalculation {
+        this.params = params
+        return this
+    }
 }
 
 

diff --git a/...tal/aambackendservice/reporting/notification/core/DefaultAddWebhookSubscriptionUseCase.kt b/...tal/aambackendservice/reporting/notification/core/DefaultAddWebhookSubscriptionUseCase.kt
@@ -24,6 +24,8 @@ class DefaultAddWebhookSubscriptionUseCase(
                     createReportCalculationUseCase.startReportCalculation(
                         CreateReportCalculationRequest(
                             report = report,
+                            from = null,
+                            to = null
                         )
                     ).flatMap {
                         Mono.just(Unit)

diff --git a/...gital/aambackendservice/reporting/report/core/DefaultReportDocumentChangeEventConsumer.kt b/...gital/aambackendservice/reporting/report/core/DefaultReportDocumentChangeEventConsumer.kt
@@ -51,7 +51,9 @@ class DefaultReportDocumentChangeEventConsumer(
 
                     return createReportCalculationUseCase.startReportCalculation(
                         request = CreateReportCalculationRequest(
-                            report = DomainReference(reportRef)
+                            report = DomainReference(reportRef),
+                            from = null,
+                            to = null,
                         )
                     ).flatMap { Mono.empty() }
                 }
@@ -74,7 +76,9 @@ class DefaultReportDocumentChangeEventConsumer(
                                 createReportCalculationUseCase
                                     .startReportCalculation(
                                         request = CreateReportCalculationRequest(
-                                            report = report
+                                            report = report,
+                                            from = null,
+                                            to = null,
                                         )
                                     )
                             }

diff --git a/.../src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/di/SqsConfiguration.kt b/.../src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/di/SqsConfiguration.kt
@@ -12,7 +12,12 @@ class SqsConfiguration {
     @Bean(name = ["sqs-client"])
     fun sqsWebClient(configuration: SqsClientConfiguration): WebClient {
         val clientBuilder =
-            WebClient.builder().baseUrl(configuration.basePath)
+            WebClient.builder()
+                .codecs {
+                    it.defaultCodecs()
+                        .maxInMemorySize(configuration.maxInMemorySizeInMegaBytes * 1024 * 1024)
+                }
+                .baseUrl(configuration.basePath)
                 .defaultHeaders {
                     it.setBasicAuth(
                         configuration.basicAuthUsername,
@@ -28,4 +33,5 @@ class SqsClientConfiguration(
     val basePath: String,
     val basicAuthUsername: String,
     val basicAuthPassword: String,
+    val maxInMemorySizeInMegaBytes: Int = 16,
 )
diff --git a/.../src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsQueryStorage.kt b/.../src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsQueryStorage.kt
@@ -49,8 +49,9 @@ class SqsQueryStorage(
                         }
                     }
             }
-            .doOnError {
+            .onErrorResume {
                 logger.error("[SqsQueryStorage]: ${it.localizedMessage}", it)
+                Mono.error(InvalidArgumentException(it.localizedMessage))
             }
     }
 }
diff --git a/...src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsSchemaService.kt b/...src/main/kotlin/com/aamdigital/aambackendservice/reporting/report/sqs/SqsSchemaService.kt
@@ -190,7 +190,7 @@ class SqsSchemaService(
 
     private fun parseEntityConfig(entityKey: String, config: AppConfigEntry): EntityType {
         return EntityType(
-            label = config.label ?: entityKey.split(":")[1],
+            label = entityKey.split(":")[1],
             attributes = config.attributes.orEmpty().map {
                 EntityAttribute(
                     name = it.key,

diff --git a/...l/aambackendservice/reporting/reportcalculation/controller/ReportCalculationController.kt b/...l/aambackendservice/reporting/reportcalculation/controller/ReportCalculationController.kt
@@ -9,13 +9,20 @@ import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage
 import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationRequest
 import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationResult
 import com.aamdigital.aambackendservice.reporting.reportcalculation.core.CreateReportCalculationUseCase
+import com.aamdigital.aambackendservice.reporting.reportcalculation.dto.ReportCalculationDto
+import com.aamdigital.aambackendservice.reporting.reportcalculation.dto.ReportDataDto
+import org.springframework.format.annotation.DateTimeFormat
 import org.springframework.validation.annotation.Validated
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.PathVariable
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
 import org.springframework.web.bind.annotation.RestController
 import reactor.core.publisher.Mono
+import java.time.ZoneOffset
+import java.time.format.DateTimeFormatter
+import java.util.*
 
 @RestController
 @RequestMapping("/v1/reporting/report-calculation")
@@ -26,7 +33,9 @@ class ReportCalculationController(
 ) {
     @PostMapping("/report/{reportId}")
     fun startCalculation(
-        @PathVariable reportId: String
+        @PathVariable reportId: String,
+        @RequestParam @DateTimeFormat(pattern = "yyyy-MM-dd") from: Date?,
+        @RequestParam @DateTimeFormat(pattern = "yyyy-MM-dd") to: Date?,
     ): Mono<DomainReference> {
         return reportingStorage.fetchReport(DomainReference(id = reportId))
             .flatMap { reportOptional ->
@@ -36,7 +45,11 @@ class ReportCalculationController(
 
                 createReportCalculationUseCase.startReportCalculation(
                     CreateReportCalculationRequest(
-                        report = DomainReference(report.id)
+                        report = DomainReference(report.id),
+                        from = from?.toInstant()?.atOffset(ZoneOffset.UTC)
+                            ?.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")),
+                        to = to?.toInstant()?.atOffset(ZoneOffset.UTC)
+                            ?.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))
                     )
                 ).handle { result, sink ->
                     when (result) {
@@ -53,14 +66,16 @@ class ReportCalculationController(
     @GetMapping("/report/{reportId}")
     fun fetchReportCalculations(
         @PathVariable reportId: String
-    ): Mono<List<ReportCalculation>> {
-        return reportingStorage.fetchCalculations(DomainReference(id = reportId))
+    ): Mono<List<ReportCalculationDto>> {
+        return reportingStorage.fetchCalculations(DomainReference(id = reportId)).map { calculations ->
+            calculations.map { toDto(it) }
+        }
     }
 
     @GetMapping("/{calculationId}")
     fun fetchReportCalculation(
         @PathVariable calculationId: String
-    ): Mono<ReportCalculation> {
+    ): Mono<ReportCalculationDto> {
         return reportingStorage.fetchCalculation(DomainReference(id = calculationId))
             .map { calculationOptional ->
                 val calculation = calculationOptional.orElseThrow {
@@ -69,14 +84,14 @@ class ReportCalculationController(
 
                 // TODO Auth check
 
-                calculation
+                toDto(calculation)
             }
     }
 
     @GetMapping("/{calculationId}/data")
     fun fetchReportCalculationData(
         @PathVariable calculationId: String
-    ): Mono<ReportData> {
+    ): Mono<ReportDataDto> {
         return reportingStorage.fetchData(DomainReference(id = calculationId))
             .map { calculationOptional ->
                 val calculation = calculationOptional.orElseThrow {
@@ -85,8 +100,24 @@ class ReportCalculationController(
 
                 // TODO Auth check
 
-                calculation
+                toDto(calculation)
             }
     }
 
+    private fun toDto(it: ReportData): ReportDataDto = ReportDataDto(
+        id = it.id,
+        report = it.report,
+        calculation = it.calculation,
+        data = it.data,
+    )
+
+    private fun toDto(it: ReportCalculation): ReportCalculationDto = ReportCalculationDto(
+        id = it.id,
+        report = it.report,
+        status = it.status,
+        startDate = it.startDate,
+        endDate = it.endDate,
+        params = it.params,
+        outcome = it.outcome,
+    )
 }
diff --git a/...ital/aambackendservice/reporting/reportcalculation/core/CreateReportCalculationUseCase.kt b/...ital/aambackendservice/reporting/reportcalculation/core/CreateReportCalculationUseCase.kt
@@ -7,6 +7,8 @@ import reactor.core.publisher.Mono
 
 data class CreateReportCalculationRequest(
     val report: DomainReference,
+    val from: String?,
+    val to: String?,
 )
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION)

diff --git a/...mbackendservice/reporting/reportcalculation/core/DefaultCreateReportCalculationUseCase.kt b/...mbackendservice/reporting/reportcalculation/core/DefaultCreateReportCalculationUseCase.kt
@@ -2,6 +2,7 @@ package com.aamdigital.aambackendservice.reporting.reportcalculation.core
 
 import com.aamdigital.aambackendservice.domain.DomainReference
 import com.aamdigital.aambackendservice.reporting.domain.ReportCalculation
+import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationParams
 import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationStatus
 import com.aamdigital.aambackendservice.reporting.report.core.ReportingStorage
 import org.springframework.stereotype.Service
@@ -17,6 +18,10 @@ class DefaultCreateReportCalculationUseCase(
             id = "ReportCalculation:${UUID.randomUUID()}",
             report = request.report,
             status = ReportCalculationStatus.PENDING,
+            params = ReportCalculationParams(
+                from = request.from,
+                to = request.to,
+            )
         )
 
         return reportingStorage.storeCalculation(calculation)

diff --git a/...kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/dto/ControllerDtos.kt b/...kotlin/com/aamdigital/aambackendservice/reporting/reportcalculation/dto/ControllerDtos.kt
@@ -0,0 +1,37 @@
+package com.aamdigital.aambackendservice.reporting.reportcalculation.dto
+
+import com.aamdigital.aambackendservice.domain.DomainReference
+import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationOutcome
+import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationParams
+import com.aamdigital.aambackendservice.reporting.domain.ReportCalculationStatus
+import com.fasterxml.jackson.databind.ObjectMapper
+import java.security.MessageDigest
+
+/**
+ * This is the interface shared to external users of the API endpoints.
+ */
+data class ReportCalculationDto(
+    val id: String,
+    val report: DomainReference,
+    var status: ReportCalculationStatus,
+    var startDate: String? = null,
+    var endDate: String? = null,
+    var params: ReportCalculationParams?,
+    var outcome: ReportCalculationOutcome? = null,
+)
+
+data class ReportDataDto(
+    val id: String,
+    val report: DomainReference,
+    val calculation: DomainReference,
+    var data: List<*>,
+) {
+    @OptIn(ExperimentalStdlibApi::class)
+    fun getDataHash(): String {
+        val mapper = ObjectMapper()
+        val md = MessageDigest.getInstance("SHA-256")
+        val input = mapper.writeValueAsString(data).toByteArray()
+        val bytes = md.digest(input)
+        return bytes.toHexString()
+    }
+}
diff --git a/...main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportingStorage.kt b/...main/kotlin/com/aamdigital/aambackendservice/reporting/storage/DefaultReportingStorage.kt
@@ -145,6 +145,7 @@ class DefaultReportingStorage(
         status = entity.doc.status,
         startDate = entity.doc.startDate,
         endDate = entity.doc.endDate,
+        params = entity.doc.params,
         outcome = entity.doc.outcome
     )
 }
diff --git a/application/aam-backend-service/src/main/resources/application.yaml b/application/aam-backend-service/src/main/resources/application.yaml
@@ -23,6 +23,13 @@ management:
           - info
           - health
 
+sqs-client-configuration:
+  max-in-memory-size-in-mega-bytes: 16
+
+
+couch-db-client-configuration:
+  max-in-memory-size-in-mega-bytes: 16
+
 ---
 
 spring:
@@ -67,7 +74,7 @@ sqs-client-configuration:
   basic-auth-password: docker
 
 database-change-detection:
-  enabled: true
+  enabled: false
 
 report-calculation-processor:
   enabled: true

diff --git a/...st/resources/cucumber/features/reporting/reportcalculation/fetch-calculation-data.feature b/...st/resources/cucumber/features/reporting/reportcalculation/fetch-calculation-data.feature
@@ -11,7 +11,7 @@ Feature: the report calculation data endpoint persist to database
         Then the client receives an json object
         Then the client receives status code of 200
         Then the client receives value 22cfbb91fcbff5a2e0755cc26567a81078898dfe939d07c9745149f45863ca31 for property dataHash
-        Then the client receives value ReportData:1 for property _id
+        Then the client receives value ReportData:1 for property id
 
     # ReportCalculation not available
     Scenario: client makes call to GET /reporting/report-calculation/ReportCalculation:42/data and receives NotFound

diff --git a/.../resources/cucumber/features/reporting/reportcalculation/fetch-report-calculation.feature b/.../resources/cucumber/features/reporting/reportcalculation/fetch-report-calculation.feature
@@ -39,7 +39,7 @@ Feature: the report calculation endpoint persist to database
         Then the client receives an json object
         Then the client receives status code of 200
         Then the client receives value FINISHED_SUCCESS for property status
-        Then the client receives value ReportCalculation:1 for property _id
+        Then the client receives value ReportCalculation:1 for property id
 
     Scenario: client makes call to GET /reporting/report-calculation/ReportCalculation:42 and receives NotFound
         Given database app is created