From 36eaa0331fb5c6596b1f46b05273fcade69a3bdb Mon Sep 17 00:00:00 2001 From: Rich Birch Date: Fri, 22 Nov 2024 10:52:16 +0000 Subject: [PATCH 1/5] DRTII-1668 Switch flight exports to streamed from fast source when non-point in time --- project/Settings.scala | 2 + .../exports/CsvFileStreaming.scala | 2 +- .../exports/FlightsExportController.scala | 57 ++++++++++--------- .../flights/templates/FlightsExport.scala | 5 +- 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/project/Settings.scala b/project/Settings.scala index adb97349b..487d866dd 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -174,6 +174,8 @@ object Settings { "com.typesafe.akka" %% "akka-http-xml" % akkaHttp, "com.typesafe.akka" %% "akka-stream" % akka, + "org.apache.pekko" %% "pekko-stream" % "1.0.1", + "com.typesafe.play" %% "twirl-api" % twirlApi, "com.typesafe.slick" %% "slick" % slick, diff --git a/server/src/main/scala/controllers/application/exports/CsvFileStreaming.scala b/server/src/main/scala/controllers/application/exports/CsvFileStreaming.scala index af6bcb800..fd24ffadd 100644 --- a/server/src/main/scala/controllers/application/exports/CsvFileStreaming.scala +++ b/server/src/main/scala/controllers/application/exports/CsvFileStreaming.scala @@ -4,7 +4,7 @@ import akka.NotUsed import akka.stream.scaladsl.Source import akka.util.ByteString import play.api.http.{HttpEntity, Writeable} -import play.api.mvc.{ResponseHeader, Result} +import play.api.mvc._ import uk.gov.homeoffice.drt.ports.PortCode import uk.gov.homeoffice.drt.ports.Terminals.Terminal import uk.gov.homeoffice.drt.time.TimeZoneHelper.europeLondonTimeZone diff --git a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala index c2d24b7f3..743b79af7 100644 --- a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala +++ b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala @@ -5,17 +5,19 @@ import actors.persistent.arrivals.{AclForecastArrivalsActor, CiriumLiveArrivalsA import akka.NotUsed import akka.pattern.ask import akka.stream.scaladsl.Source +import akka.util.ByteString import com.google.inject.Inject import controllers.application.AuthController import controllers.application.exports.CsvFileStreaming.{makeFileName, sourceToCsvResponse} import drt.shared.CrunchApi.MillisSinceEpoch import passengersplits.parsing.VoyageManifestParser.VoyageManifests +import play.api.http.{HttpChunk, HttpEntity, Writeable} import play.api.mvc._ import services.exports.flights.ArrivalFeedExport import services.exports.flights.templates._ import services.exports.{FlightExports, GeneralExport} import uk.gov.homeoffice.drt.actor.commands.Commands.GetState -import uk.gov.homeoffice.drt.arrivals.{ApiFlightWithSplits, FlightsWithSplits} +import uk.gov.homeoffice.drt.arrivals.FlightsWithSplits import uk.gov.homeoffice.drt.auth.LoggedInUser import uk.gov.homeoffice.drt.auth.Roles.{ApiView, ArrivalSource, ArrivalsAndSplitsView, SuperAdmin} import uk.gov.homeoffice.drt.crunchsystem.DrtSystemInterface @@ -48,8 +50,8 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste maybeDate match { case Some(localDate) => ctrl.applicationService.redListUpdatesActor.ask(GetState).mapTo[RedListUpdates].flatMap { redListUpdates => - val export = exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(localDate, localDate, terminals) - flightsRequestToCsv(Option(pointInTime), export) + requestToCsvStream(Option(pointInTime), exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(localDate, localDate, terminals)) + .map(streamExport(terminals, localDate, localDate, _, "flights")) } case _ => Future(BadRequest("Invalid date format for export day.")) @@ -108,13 +110,16 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste exportTerminalDateRange: (LoggedInUser, PortCode, RedListUpdates) => (LocalDate, LocalDate, Seq[Terminal]) => FlightsExport, ): Action[AnyContent] = { + + Action.async { request => val user = ctrl.getLoggedInUser(config, request.headers, request.session) (LocalDate.parse(startLocalDateString), LocalDate.parse(endLocalDateString)) match { case (Some(start), Some(end)) => ctrl.applicationService.redListUpdatesActor.ask(GetState).mapTo[RedListUpdates].flatMap { redListUpdates => - flightsRequestToCsv(None, exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(start, end, terminals)) + requestToCsvStream(None, exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(start, end, terminals)) + .map(streamExport(terminals, start, end, _, "flights")) } case _ => Future(BadRequest("Invalid date format for start or end date")) @@ -122,6 +127,21 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste } } + private def streamExport(terminals: Seq[Terminal], start: LocalDate, end: LocalDate, stream: Source[String, NotUsed], exportName: String): Result = { + implicit val writeable: Writeable[String] = Writeable(ByteString.fromString, Option("text/csv")) + + val header = ResponseHeader(OK) + val fileName = makeFileName(exportName, terminals, start, end, airportConfig.portCode) + ".csv" + + Result( + header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName))), + body = HttpEntity.Chunked( + stream.map(c => HttpChunk.Chunk(writeable.transform(c))), + fileMimeTypes.forFileName(fileName) + ) + ) + } + private def doExportForDateRangeLegacy(startLocalDateString: String, endLocalDateString: String, terminals: Seq[Terminal], @@ -142,9 +162,7 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste } } - private def flightsRequestToCsv(maybePointInTime: Option[MillisSinceEpoch], - `export`: FlightsExport, - ): Future[Result] = { + private def requestToCsvStream(maybePointInTime: Option[MillisSinceEpoch], `export`: FlightsExport): Future[Source[String, NotUsed]] = { val eventualFlightsByDate = maybePointInTime match { case Some(pointInTime) => val requestStart = SDate(`export`.start).millisSinceEpoch @@ -160,13 +178,10 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste eventualFlightsByDate.map { flightsStream => - val flightsAndManifestsStream = flightsStream.mapAsync(1) { case (d, flights) => + export.csvStream(flightsStream.mapAsync(1) { case (d, flights) => val sortedFlights = flights.toSeq.sortBy(_.apiFlight.PcpTime.getOrElse(0L)) ctrl.applicationService.manifestsProvider(d, d).map(_._2).runFold(VoyageManifests.empty)(_ ++ _).map(m => (sortedFlights, m)) - } - val csvStream = export.csvStream(flightsAndManifestsStream) - val fileName = makeFileName("flights", export.terminals, export.start, export.end, airportConfig.portCode) + ".csv" - tryCsvResponse(csvStream, fileName) + }) } } @@ -229,24 +244,14 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste val persistenceId = feedSourceToPersistenceId(fs) val arrivalsExport = ArrivalFeedExport(ctrl.feedService.paxFeedSourceOrder) val startDate = SDate(startPit) - val numberOfDays = startDate.getLocalLastMidnight.daysBetweenInclusive(SDate(endPit)) + val endDate = SDate(endPit) + val numberOfDays = startDate.getLocalLastMidnight.daysBetweenInclusive(endDate) val csvDataSource = arrivalsExport.flightsDataSource(startDate, numberOfDays, terminal, fs, persistenceId) - val periodString = if (numberOfDays > 1) - s"${startDate.getLocalLastMidnight.toISODateOnly}-to-${SDate(endPit).getLocalLastMidnight.toISODateOnly}" - else - startDate.getLocalLastMidnight.toISODateOnly - - val fileName = s"${ - airportConfig.portCode - }-$terminal-$feedSourceString-$periodString.csv" - - val byteStringStream = csvDataSource.collect { - case Some(s) => s - } + val stream = csvDataSource.collect { case Some(s) => s } - sourceToCsvResponse(byteStringStream, fileName) + streamExport(Seq(terminal), startDate.toLocalDate, endDate.toLocalDate, stream, s"flights-$feedSourceString-$terminal") case None => NotFound(s"Unknown feed source $feedSourceString") diff --git a/server/src/main/scala/services/exports/flights/templates/FlightsExport.scala b/server/src/main/scala/services/exports/flights/templates/FlightsExport.scala index 4c40a447a..74c24e74d 100644 --- a/server/src/main/scala/services/exports/flights/templates/FlightsExport.scala +++ b/server/src/main/scala/services/exports/flights/templates/FlightsExport.scala @@ -34,9 +34,7 @@ trait FlightsExport { def csvStream(flightsStream: Source[(Iterable[ApiFlightWithSplits], VoyageManifests), NotUsed]): Source[String, NotUsed] = filterAndSort(flightsStream) - .map { case (fws, maybeManifest) => - flightToCsvRow(fws, maybeManifest) + "\n" - } + .map { case (fws, maybeManifest) => flightToCsvRow(fws, maybeManifest) + "\n" } .prepend(Source(List(headings + "\n"))) private def filterAndSort(flightsStream: Source[(Iterable[ApiFlightWithSplits], VoyageManifests), NotUsed], @@ -51,5 +49,4 @@ trait FlightsExport { (fws, maybeManifest) } } - } From 81bc8c49738abf34d01e145ec2ef3e1227d1cac7 Mon Sep 17 00:00:00 2001 From: Rich Birch Date: Fri, 22 Nov 2024 11:36:47 +0000 Subject: [PATCH 2/5] DRTII-1668 Try disabling proxy request buffering --- .../exports/FlightsExportController.scala | 37 +++++++++---------- .../main/scala/services/exports/Exports.scala | 37 ++++++++++++++++++- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala index 743b79af7..31cc4d9e9 100644 --- a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala +++ b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala @@ -5,14 +5,13 @@ import actors.persistent.arrivals.{AclForecastArrivalsActor, CiriumLiveArrivalsA import akka.NotUsed import akka.pattern.ask import akka.stream.scaladsl.Source -import akka.util.ByteString import com.google.inject.Inject import controllers.application.AuthController import controllers.application.exports.CsvFileStreaming.{makeFileName, sourceToCsvResponse} import drt.shared.CrunchApi.MillisSinceEpoch import passengersplits.parsing.VoyageManifestParser.VoyageManifests -import play.api.http.{HttpChunk, HttpEntity, Writeable} import play.api.mvc._ +import services.exports.Exports.streamExport import services.exports.flights.ArrivalFeedExport import services.exports.flights.templates._ import services.exports.{FlightExports, GeneralExport} @@ -51,7 +50,7 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste case Some(localDate) => ctrl.applicationService.redListUpdatesActor.ask(GetState).mapTo[RedListUpdates].flatMap { redListUpdates => requestToCsvStream(Option(pointInTime), exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(localDate, localDate, terminals)) - .map(streamExport(terminals, localDate, localDate, _, "flights")) + .map(streamExport(airportConfig.portCode, terminals, localDate, localDate, _, "flights")) } case _ => Future(BadRequest("Invalid date format for export day.")) @@ -119,7 +118,7 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste case (Some(start), Some(end)) => ctrl.applicationService.redListUpdatesActor.ask(GetState).mapTo[RedListUpdates].flatMap { redListUpdates => requestToCsvStream(None, exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(start, end, terminals)) - .map(streamExport(terminals, start, end, _, "flights")) + .map(streamExport(airportConfig.portCode, terminals, start, end, _, "flights")) } case _ => Future(BadRequest("Invalid date format for start or end date")) @@ -127,20 +126,20 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste } } - private def streamExport(terminals: Seq[Terminal], start: LocalDate, end: LocalDate, stream: Source[String, NotUsed], exportName: String): Result = { - implicit val writeable: Writeable[String] = Writeable(ByteString.fromString, Option("text/csv")) - - val header = ResponseHeader(OK) - val fileName = makeFileName(exportName, terminals, start, end, airportConfig.portCode) + ".csv" - - Result( - header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName))), - body = HttpEntity.Chunked( - stream.map(c => HttpChunk.Chunk(writeable.transform(c))), - fileMimeTypes.forFileName(fileName) - ) - ) - } +// def streamExport(terminals: Seq[Terminal], start: LocalDate, end: LocalDate, stream: Source[String, NotUsed], exportName: String): Result = { +// implicit val writeable: Writeable[String] = Writeable(ByteString.fromString, Option("text/csv")) +// +// val header = ResponseHeader(OK) +// val fileName = makeFileName(exportName, terminals, start, end, airportConfig.portCode) + ".csv" +// +// Result( +// header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName))), +// body = HttpEntity.Chunked( +// stream.map(c => HttpChunk.Chunk(writeable.transform(c))), +// fileMimeTypes.forFileName(fileName) +// ) +// ) +// } private def doExportForDateRangeLegacy(startLocalDateString: String, endLocalDateString: String, @@ -251,7 +250,7 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste val stream = csvDataSource.collect { case Some(s) => s } - streamExport(Seq(terminal), startDate.toLocalDate, endDate.toLocalDate, stream, s"flights-$feedSourceString-$terminal") + streamExport(airportConfig.portCode, Seq(terminal), startDate.toLocalDate, endDate.toLocalDate, stream, s"flights-$feedSourceString") case None => NotFound(s"Unknown feed source $feedSourceString") diff --git a/server/src/main/scala/services/exports/Exports.scala b/server/src/main/scala/services/exports/Exports.scala index c0c4b4893..5f096f087 100644 --- a/server/src/main/scala/services/exports/Exports.scala +++ b/server/src/main/scala/services/exports/Exports.scala @@ -1,12 +1,23 @@ package services.exports +import akka.NotUsed +import akka.stream.scaladsl.Source +import akka.util.ByteString +import controllers.application.exports.CsvFileStreaming.makeFileName import drt.shared.CrunchApi.MillisSinceEpoch import org.slf4j.{Logger, LoggerFactory} +import play.api.http.Status.OK +import play.api.http.{HttpChunk, HttpEntity, Writeable} +import play.api.mvc.{ResponseHeader, Result, Results} +import play.mvc.StaticFileMimeTypes.fileMimeTypes import uk.gov.homeoffice.drt.arrivals.{ApiFlightWithSplits, Splits} -import uk.gov.homeoffice.drt.ports.SplitRatiosNs -import uk.gov.homeoffice.drt.time.SDate +import uk.gov.homeoffice.drt.ports.{PortCode, SplitRatiosNs} +import uk.gov.homeoffice.drt.ports.Terminals.Terminal +import uk.gov.homeoffice.drt.time.{LocalDate, SDate} import uk.gov.homeoffice.drt.time.TimeZoneHelper.europeLondonTimeZone +import scala.compat.java8.OptionConverters.RichOptionalGeneric + object Exports { val log: Logger = LoggerFactory.getLogger(getClass) @@ -22,4 +33,26 @@ object Exports { s.splits.map(s => (s"API Actual - ${s.paxTypeAndQueue.displayName}", s.paxCount)) } .flatten + + def streamExport(portCode: PortCode, + terminals: Seq[Terminal], + start: LocalDate, + end: LocalDate, + stream: Source[String, NotUsed], + exportName: String): Result = { + implicit val writeable: Writeable[String] = Writeable(ByteString.fromString, Option("text/csv")) + + val header = ResponseHeader(OK) + val disableNginxProxyBuffering = "X-Accel-Buffering" -> "no" + val fileName = makeFileName(exportName, terminals, start, end, portCode) + ".csv" + + Result( + header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName)) ++ disableNginxProxyBuffering), + body = HttpEntity.Chunked( + stream.map(c => HttpChunk.Chunk(writeable.transform(c))), + fileMimeTypes.forFileName(fileName).asScala + ) + ) + } + } From defc5d2b5bf6b6d67dfea88a221bfe008d11881a Mon Sep 17 00:00:00 2001 From: Rich Birch Date: Fri, 22 Nov 2024 11:45:46 +0000 Subject: [PATCH 3/5] DRTII-1668 fix header setting --- server/src/main/scala/services/exports/Exports.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/scala/services/exports/Exports.scala b/server/src/main/scala/services/exports/Exports.scala index 5f096f087..b02dbc8a8 100644 --- a/server/src/main/scala/services/exports/Exports.scala +++ b/server/src/main/scala/services/exports/Exports.scala @@ -47,7 +47,7 @@ object Exports { val fileName = makeFileName(exportName, terminals, start, end, portCode) + ".csv" Result( - header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName)) ++ disableNginxProxyBuffering), + header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName)) ++ Option(disableNginxProxyBuffering)), body = HttpEntity.Chunked( stream.map(c => HttpChunk.Chunk(writeable.transform(c))), fileMimeTypes.forFileName(fileName).asScala From 8cc0dadf5badeabaae9af56e43c813a1d49c9ef1 Mon Sep 17 00:00:00 2001 From: Rich Birch Date: Fri, 22 Nov 2024 14:08:39 +0000 Subject: [PATCH 4/5] DRTII-1668 Tidying --- project/Settings.scala | 2 -- .../exports/FlightsExportController.scala | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/project/Settings.scala b/project/Settings.scala index 487d866dd..adb97349b 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -174,8 +174,6 @@ object Settings { "com.typesafe.akka" %% "akka-http-xml" % akkaHttp, "com.typesafe.akka" %% "akka-stream" % akka, - "org.apache.pekko" %% "pekko-stream" % "1.0.1", - "com.typesafe.play" %% "twirl-api" % twirlApi, "com.typesafe.slick" %% "slick" % slick, diff --git a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala index 31cc4d9e9..1a8babb87 100644 --- a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala +++ b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala @@ -126,21 +126,6 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste } } -// def streamExport(terminals: Seq[Terminal], start: LocalDate, end: LocalDate, stream: Source[String, NotUsed], exportName: String): Result = { -// implicit val writeable: Writeable[String] = Writeable(ByteString.fromString, Option("text/csv")) -// -// val header = ResponseHeader(OK) -// val fileName = makeFileName(exportName, terminals, start, end, airportConfig.portCode) + ".csv" -// -// Result( -// header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName))), -// body = HttpEntity.Chunked( -// stream.map(c => HttpChunk.Chunk(writeable.transform(c))), -// fileMimeTypes.forFileName(fileName) -// ) -// ) -// } - private def doExportForDateRangeLegacy(startLocalDateString: String, endLocalDateString: String, terminals: Seq[Terminal], From 5b7e28915986cb47f0cdc64fb602b7340aa14a2d Mon Sep 17 00:00:00 2001 From: Rich Birch Date: Tue, 26 Nov 2024 09:11:34 +0000 Subject: [PATCH 5/5] DRTII-1574 Desks and queues exports streaming --- client/package-lock.json | 2 +- .../exports/DesksExportController.scala | 90 +++++++------------ .../exports/FlightsExportController.scala | 5 +- 3 files changed, 36 insertions(+), 61 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 76c6cf557..7ca9e89dc 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1,5 +1,5 @@ { - "name": "test", + "name": "main", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/server/src/main/scala/controllers/application/exports/DesksExportController.scala b/server/src/main/scala/controllers/application/exports/DesksExportController.scala index 7f58dbed6..80318c74b 100644 --- a/server/src/main/scala/controllers/application/exports/DesksExportController.scala +++ b/server/src/main/scala/controllers/application/exports/DesksExportController.scala @@ -4,129 +4,107 @@ import akka.NotUsed import akka.stream.scaladsl.Source import com.google.inject.Inject import controllers.application.AuthController -import controllers.application.exports.CsvFileStreaming.{makeFileName, sourceToCsvResponse} import drt.shared.CrunchApi.MillisSinceEpoch import drt.shared.ErrorResponse import play.api.mvc.{Action, AnyContent, ControllerComponents, Request} +import services.exports.Exports.streamExport import services.exports.StreamingDesksExport -import uk.gov.homeoffice.drt.time.SDate import uk.gov.homeoffice.drt.auth.Roles.DesksAndQueuesView import uk.gov.homeoffice.drt.crunchsystem.DrtSystemInterface import uk.gov.homeoffice.drt.ports.Terminals.Terminal -import uk.gov.homeoffice.drt.time.{LocalDate, SDateLike} +import uk.gov.homeoffice.drt.time.{LocalDate, SDate, SDateLike} import upickle.default.write -import scala.concurrent.Future -import scala.util.{Failure, Success, Try} +import scala.util.{Success, Try} class DesksExportController @Inject()(cc: ControllerComponents, ctrl: DrtSystemInterface) extends AuthController(cc, ctrl) { def exportDesksAndQueuesRecsAtPointInTimeCSV(localDate: String, pointInTime: String, terminalName: String): Action[AnyContent] = - Action.async { request => - authByRole(DesksAndQueuesView) { + authByRole(DesksAndQueuesView) { + Action { request => (LocalDate.parse(localDate), Try(SDate(pointInTime.toLong))) match { case (Some(ld), Success(pit)) => val viewDay = SDate(ld) val start = viewDay val end = viewDay.getLocalNextMidnight.addMinutes(-1) - exportBetweenTimestampsCSV( - deskRecsExportStreamForTerminalDates(pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request)), - makeFileName(s"desks-and-queues-recs-at-${pit.toISOString}-for", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv", - ) + val stream = deskRecsExportStreamForTerminalDates( + pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request)) + streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), ld, ld, stream, s"desks-and-queues-recs-at-${pit.toISOString}-for") case _ => - Action(BadRequest(write(ErrorResponse("Invalid date format")))) + BadRequest(write(ErrorResponse("Invalid date format"))) } - }(request) + } } - private def periodMinutes(request: Request[AnyContent]) = { + private def periodMinutes(request: Request[AnyContent]): Int = request.getQueryString("period-minutes").map(_.toInt).getOrElse(15) - } def exportDesksAndQueuesRecsBetweenTimeStampsCSV(startLocalDate: String, endLocalDate: String, terminalName: String): Action[AnyContent] = - Action.async { request => - authByRole(DesksAndQueuesView) { + authByRole(DesksAndQueuesView) { + Action { request => (LocalDate.parse(startLocalDate), LocalDate.parse(endLocalDate)) match { case (Some(startLD), Some(endLD)) => val start = SDate(startLD) val end = SDate(endLD).getLocalNextMidnight.addMinutes(-1) - exportBetweenTimestampsCSV( - deskRecsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request)), - makeFileName("desks-and-queues-recs", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv", - ) + val stream = deskRecsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request)) + streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), startLD, endLD, stream, "desks-and-queues-recs") case _ => - Action(BadRequest(write(ErrorResponse("Invalid date format")))) + BadRequest(write(ErrorResponse("Invalid date format"))) } - }(request) + } } def exportDesksAndQueuesDepsAtPointInTimeCSV(localDate: String, pointInTime: String, terminalName: String ): Action[AnyContent] = - Action.async { request => - authByRole(DesksAndQueuesView) { + authByRole(DesksAndQueuesView) { + Action { request => (LocalDate.parse(localDate), Try(SDate(pointInTime.toLong))) match { case (Some(ld), Success(pit)) => val start = SDate(ld) val end = start.getLocalNextMidnight.addMinutes(-1) - exportBetweenTimestampsCSV( - deploymentsExportStreamForTerminalDates(pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request)), - makeFileName(s"desks-and-queues-deps-at-${pit.toISOString}-for", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv", - ) + val stream = deploymentsExportStreamForTerminalDates( + pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request)) + streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), ld, ld, stream, s"desks-and-queues-deps-at-${pit.toISOString}-for") case _ => - Action(BadRequest(write(ErrorResponse("Invalid date format")))) + BadRequest(write(ErrorResponse("Invalid date format"))) } - }(request) + } } def exportDesksAndQueuesDepsBetweenTimeStampsCSV(startLocalDate: String, endLocalDate: String, terminalName: String): Action[AnyContent] = - Action.async { request => - authByRole(DesksAndQueuesView) { + authByRole(DesksAndQueuesView) { + Action { request => (LocalDate.parse(startLocalDate), LocalDate.parse(endLocalDate)) match { case (Some(startLD), Some(endLD)) => val start = SDate(startLD) val end = SDate(endLD).getLocalNextMidnight.addMinutes(-1) - exportBetweenTimestampsCSV( - deploymentsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request)), - makeFileName("desks-and-queues-deps", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv", - ) + val stream = deploymentsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request)) + streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), startLD, endLD, stream, "desks-and-queues-deps") case _ => - Action(BadRequest(write(ErrorResponse("Invalid date format")))) + BadRequest(write(ErrorResponse("Invalid date format"))) } - }(request) + } } - private def exportBetweenTimestampsCSV(exportSourceFn: () => Source[String, NotUsed], - fileName: String, - ): Action[AnyContent] = Action.async { - val exportSource: Source[String, NotUsed] = exportSourceFn() - - Try(sourceToCsvResponse(exportSource, fileName)) match { - case Success(value) => Future(value) - case Failure(t) => - log.error(s"Failed to get CSV export: ${t.getMessage}") - Future(BadRequest("Failed to get CSV export")) - } - } - private def deskRecsExportStreamForTerminalDates(pointInTime: Option[MillisSinceEpoch], start: SDateLike, end: SDateLike, terminal: Terminal, periodMinutes: Int, - ): () => Source[String, NotUsed] = - () => StreamingDesksExport.deskRecsToCSVStreamWithHeaders( + ): Source[String, NotUsed] = + StreamingDesksExport.deskRecsToCSVStreamWithHeaders( start, end, terminal, @@ -142,8 +120,8 @@ class DesksExportController @Inject()(cc: ControllerComponents, ctrl: DrtSystemI end: SDateLike, terminal: Terminal, periodMinutes: Int, - ): () => Source[String, NotUsed] = - () => StreamingDesksExport.deploymentsToCSVStreamWithHeaders( + ): Source[String, NotUsed] = + StreamingDesksExport.deploymentsToCSVStreamWithHeaders( start, end, terminal, diff --git a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala index 1a8babb87..ebe4e8360 100644 --- a/server/src/main/scala/controllers/application/exports/FlightsExportController.scala +++ b/server/src/main/scala/controllers/application/exports/FlightsExportController.scala @@ -108,9 +108,7 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste terminals: Seq[Terminal], exportTerminalDateRange: (LoggedInUser, PortCode, RedListUpdates) => (LocalDate, LocalDate, Seq[Terminal]) => FlightsExport, - ): Action[AnyContent] = { - - + ): Action[AnyContent] = Action.async { request => val user = ctrl.getLoggedInUser(config, request.headers, request.session) @@ -124,7 +122,6 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste Future(BadRequest("Invalid date format for start or end date")) } } - } private def doExportForDateRangeLegacy(startLocalDateString: String, endLocalDateString: String,