Skip to content

Commit

Permalink
Merge pull request #1960 from UKHomeOffice/DRTII-1668-Multi-day-fligh…
Browse files Browse the repository at this point in the history 
…ts-download-fail-on-historic-date-ranges

Drtii 1668 multi day flights download fail on historic date ranges
  • Loading branch information
@richbirch
richbirch authored Nov 26, 2024
2 parents c6fa479 + 5b7e289 commit 380a1f7
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 92 deletions.
2 changes: 1 addition & 1 deletion client/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion server/src/main/scala/controllers/application/exports/CsvFileStreaming.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import akka.NotUsed
import akka.stream.scaladsl.Source
import akka.util.ByteString
import play.api.http.{HttpEntity, Writeable}
import play.api.mvc.{ResponseHeader, Result}
import play.api.mvc._
import uk.gov.homeoffice.drt.ports.PortCode
import uk.gov.homeoffice.drt.ports.Terminals.Terminal
import uk.gov.homeoffice.drt.time.TimeZoneHelper.europeLondonTimeZone
Expand Down
90 changes: 34 additions & 56 deletions server/src/main/scala/controllers/application/exports/DesksExportController.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,129 +4,107 @@ import akka.NotUsed
import akka.stream.scaladsl.Source
import com.google.inject.Inject
import controllers.application.AuthController
import controllers.application.exports.CsvFileStreaming.{makeFileName, sourceToCsvResponse}
import drt.shared.CrunchApi.MillisSinceEpoch
import drt.shared.ErrorResponse
import play.api.mvc.{Action, AnyContent, ControllerComponents, Request}
import services.exports.Exports.streamExport
import services.exports.StreamingDesksExport
import uk.gov.homeoffice.drt.time.SDate
import uk.gov.homeoffice.drt.auth.Roles.DesksAndQueuesView
import uk.gov.homeoffice.drt.crunchsystem.DrtSystemInterface
import uk.gov.homeoffice.drt.ports.Terminals.Terminal
import uk.gov.homeoffice.drt.time.{LocalDate, SDateLike}
import uk.gov.homeoffice.drt.time.{LocalDate, SDate, SDateLike}
import upickle.default.write

import scala.concurrent.Future
import scala.util.{Failure, Success, Try}
import scala.util.{Success, Try}

class DesksExportController @Inject()(cc: ControllerComponents, ctrl: DrtSystemInterface) extends AuthController(cc, ctrl) {

def exportDesksAndQueuesRecsAtPointInTimeCSV(localDate: String,
pointInTime: String,
terminalName: String): Action[AnyContent] =
Action.async { request =>
authByRole(DesksAndQueuesView) {
authByRole(DesksAndQueuesView) {
Action { request =>
(LocalDate.parse(localDate), Try(SDate(pointInTime.toLong))) match {
case (Some(ld), Success(pit)) =>
val viewDay = SDate(ld)
val start = viewDay
val end = viewDay.getLocalNextMidnight.addMinutes(-1)

exportBetweenTimestampsCSV(
deskRecsExportStreamForTerminalDates(pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request)),
makeFileName(s"desks-and-queues-recs-at-${pit.toISOString}-for", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv",
)
val stream = deskRecsExportStreamForTerminalDates(
pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request))
streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), ld, ld, stream, s"desks-and-queues-recs-at-${pit.toISOString}-for")
case _ =>
Action(BadRequest(write(ErrorResponse("Invalid date format"))))
BadRequest(write(ErrorResponse("Invalid date format")))
}
}(request)
}
}

private def periodMinutes(request: Request[AnyContent]) = {
private def periodMinutes(request: Request[AnyContent]): Int =
request.getQueryString("period-minutes").map(_.toInt).getOrElse(15)
}

def exportDesksAndQueuesRecsBetweenTimeStampsCSV(startLocalDate: String,
endLocalDate: String,
terminalName: String): Action[AnyContent] =
Action.async { request =>
authByRole(DesksAndQueuesView) {
authByRole(DesksAndQueuesView) {
Action { request =>
(LocalDate.parse(startLocalDate), LocalDate.parse(endLocalDate)) match {
case (Some(startLD), Some(endLD)) =>
val start = SDate(startLD)
val end = SDate(endLD).getLocalNextMidnight.addMinutes(-1)

exportBetweenTimestampsCSV(
deskRecsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request)),
makeFileName("desks-and-queues-recs", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv",
)
val stream = deskRecsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request))
streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), startLD, endLD, stream, "desks-and-queues-recs")
case _ =>
Action(BadRequest(write(ErrorResponse("Invalid date format"))))
BadRequest(write(ErrorResponse("Invalid date format")))
}
}(request)
}
}

def exportDesksAndQueuesDepsAtPointInTimeCSV(localDate: String,
pointInTime: String,
terminalName: String
): Action[AnyContent] =
Action.async { request =>
authByRole(DesksAndQueuesView) {
authByRole(DesksAndQueuesView) {
Action { request =>
(LocalDate.parse(localDate), Try(SDate(pointInTime.toLong))) match {
case (Some(ld), Success(pit)) =>
val start = SDate(ld)
val end = start.getLocalNextMidnight.addMinutes(-1)

exportBetweenTimestampsCSV(
deploymentsExportStreamForTerminalDates(pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request)),
makeFileName(s"desks-and-queues-deps-at-${pit.toISOString}-for", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv",
)
val stream = deploymentsExportStreamForTerminalDates(
pointInTime = Option(pit.millisSinceEpoch), start, end, Terminal(terminalName), periodMinutes(request))
streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), ld, ld, stream, s"desks-and-queues-deps-at-${pit.toISOString}-for")
case _ =>
Action(BadRequest(write(ErrorResponse("Invalid date format"))))
BadRequest(write(ErrorResponse("Invalid date format")))
}
}(request)
}
}

def exportDesksAndQueuesDepsBetweenTimeStampsCSV(startLocalDate: String,
endLocalDate: String,
terminalName: String): Action[AnyContent] =
Action.async { request =>
authByRole(DesksAndQueuesView) {
authByRole(DesksAndQueuesView) {
Action { request =>
(LocalDate.parse(startLocalDate), LocalDate.parse(endLocalDate)) match {
case (Some(startLD), Some(endLD)) =>
val start = SDate(startLD)
val end = SDate(endLD).getLocalNextMidnight.addMinutes(-1)

exportBetweenTimestampsCSV(
deploymentsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request)),
makeFileName("desks-and-queues-deps", Seq(Terminal(terminalName)), start, end, airportConfig.portCode) + ".csv",
)
val stream = deploymentsExportStreamForTerminalDates(pointInTime = None, start, end, Terminal(terminalName), periodMinutes(request))
streamExport(airportConfig.portCode, Seq(Terminal(terminalName)), startLD, endLD, stream, "desks-and-queues-deps")
case _ =>
Action(BadRequest(write(ErrorResponse("Invalid date format"))))
BadRequest(write(ErrorResponse("Invalid date format")))
}
}(request)
}
}

private def exportBetweenTimestampsCSV(exportSourceFn: () => Source[String, NotUsed],
fileName: String,
): Action[AnyContent] = Action.async {
val exportSource: Source[String, NotUsed] = exportSourceFn()

Try(sourceToCsvResponse(exportSource, fileName)) match {
case Success(value) => Future(value)
case Failure(t) =>
log.error(s"Failed to get CSV export: ${t.getMessage}")
Future(BadRequest("Failed to get CSV export"))
}
}

private def deskRecsExportStreamForTerminalDates(pointInTime: Option[MillisSinceEpoch],
start: SDateLike,
end: SDateLike,
terminal: Terminal,
periodMinutes: Int,
): () => Source[String, NotUsed] =
() => StreamingDesksExport.deskRecsToCSVStreamWithHeaders(
): Source[String, NotUsed] =
StreamingDesksExport.deskRecsToCSVStreamWithHeaders(
start,
end,
terminal,
Expand All @@ -142,8 +120,8 @@ class DesksExportController @Inject()(cc: ControllerComponents, ctrl: DrtSystemI
end: SDateLike,
terminal: Terminal,
periodMinutes: Int,
): () => Source[String, NotUsed] =
() => StreamingDesksExport.deploymentsToCSVStreamWithHeaders(
): Source[String, NotUsed] =
StreamingDesksExport.deploymentsToCSVStreamWithHeaders(
start,
end,
terminal,
Expand Down
42 changes: 14 additions & 28 deletions server/src/main/scala/controllers/application/exports/FlightsExportController.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ import controllers.application.exports.CsvFileStreaming.{makeFileName, sourceToC
import drt.shared.CrunchApi.MillisSinceEpoch
import passengersplits.parsing.VoyageManifestParser.VoyageManifests
import play.api.mvc._
import services.exports.Exports.streamExport
import services.exports.flights.ArrivalFeedExport
import services.exports.flights.templates._
import services.exports.{FlightExports, GeneralExport}
import uk.gov.homeoffice.drt.actor.commands.Commands.GetState
import uk.gov.homeoffice.drt.arrivals.{ApiFlightWithSplits, FlightsWithSplits}
import uk.gov.homeoffice.drt.arrivals.FlightsWithSplits
import uk.gov.homeoffice.drt.auth.LoggedInUser
import uk.gov.homeoffice.drt.auth.Roles.{ApiView, ArrivalSource, ArrivalsAndSplitsView, SuperAdmin}
import uk.gov.homeoffice.drt.crunchsystem.DrtSystemInterface
Expand Down Expand Up @@ -48,8 +49,8 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste
maybeDate match {
case Some(localDate) =>
ctrl.applicationService.redListUpdatesActor.ask(GetState).mapTo[RedListUpdates].flatMap { redListUpdates =>
val export = exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(localDate, localDate, terminals)
flightsRequestToCsv(Option(pointInTime), export)
requestToCsvStream(Option(pointInTime), exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(localDate, localDate, terminals))
.map(streamExport(airportConfig.portCode, terminals, localDate, localDate, _, "flights"))
}
case _ =>
Future(BadRequest("Invalid date format for export day."))
Expand Down Expand Up @@ -107,20 +108,20 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste
terminals: Seq[Terminal],
exportTerminalDateRange: (LoggedInUser, PortCode, RedListUpdates)
=> (LocalDate, LocalDate, Seq[Terminal]) => FlightsExport,
): Action[AnyContent] = {
): Action[AnyContent] =
Action.async {
request =>
val user = ctrl.getLoggedInUser(config, request.headers, request.session)
(LocalDate.parse(startLocalDateString), LocalDate.parse(endLocalDateString)) match {
case (Some(start), Some(end)) =>
ctrl.applicationService.redListUpdatesActor.ask(GetState).mapTo[RedListUpdates].flatMap { redListUpdates =>
flightsRequestToCsv(None, exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(start, end, terminals))
requestToCsvStream(None, exportTerminalDateRange(user, airportConfig.portCode, redListUpdates)(start, end, terminals))
.map(streamExport(airportConfig.portCode, terminals, start, end, _, "flights"))
}
case _ =>
Future(BadRequest("Invalid date format for start or end date"))
}
}
}

private def doExportForDateRangeLegacy(startLocalDateString: String,
endLocalDateString: String,
Expand All @@ -142,9 +143,7 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste
}
}

private def flightsRequestToCsv(maybePointInTime: Option[MillisSinceEpoch],
`export`: FlightsExport,
): Future[Result] = {
private def requestToCsvStream(maybePointInTime: Option[MillisSinceEpoch], `export`: FlightsExport): Future[Source[String, NotUsed]] = {
val eventualFlightsByDate = maybePointInTime match {
case Some(pointInTime) =>
val requestStart = SDate(`export`.start).millisSinceEpoch
Expand All @@ -160,13 +159,10 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste

eventualFlightsByDate.map {
flightsStream =>
val flightsAndManifestsStream = flightsStream.mapAsync(1) { case (d, flights) =>
export.csvStream(flightsStream.mapAsync(1) { case (d, flights) =>
val sortedFlights = flights.toSeq.sortBy(_.apiFlight.PcpTime.getOrElse(0L))
ctrl.applicationService.manifestsProvider(d, d).map(_._2).runFold(VoyageManifests.empty)(_ ++ _).map(m => (sortedFlights, m))
}
val csvStream = export.csvStream(flightsAndManifestsStream)
val fileName = makeFileName("flights", export.terminals, export.start, export.end, airportConfig.portCode) + ".csv"
tryCsvResponse(csvStream, fileName)
})
}
}

Expand Down Expand Up @@ -229,24 +225,14 @@ class FlightsExportController @Inject()(cc: ControllerComponents, ctrl: DrtSyste
val persistenceId = feedSourceToPersistenceId(fs)
val arrivalsExport = ArrivalFeedExport(ctrl.feedService.paxFeedSourceOrder)
val startDate = SDate(startPit)
val numberOfDays = startDate.getLocalLastMidnight.daysBetweenInclusive(SDate(endPit))
val endDate = SDate(endPit)
val numberOfDays = startDate.getLocalLastMidnight.daysBetweenInclusive(endDate)

val csvDataSource = arrivalsExport.flightsDataSource(startDate, numberOfDays, terminal, fs, persistenceId)

val periodString = if (numberOfDays > 1)
s"${startDate.getLocalLastMidnight.toISODateOnly}-to-${SDate(endPit).getLocalLastMidnight.toISODateOnly}"
else
startDate.getLocalLastMidnight.toISODateOnly

val fileName = s"${
airportConfig.portCode
}-$terminal-$feedSourceString-$periodString.csv"

val byteStringStream = csvDataSource.collect {
case Some(s) => s
}
val stream = csvDataSource.collect { case Some(s) => s }

sourceToCsvResponse(byteStringStream, fileName)
streamExport(airportConfig.portCode, Seq(terminal), startDate.toLocalDate, endDate.toLocalDate, stream, s"flights-$feedSourceString")

case None =>
NotFound(s"Unknown feed source $feedSourceString")
Expand Down
37 changes: 35 additions & 2 deletions server/src/main/scala/services/exports/Exports.scala
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,23 @@
package services.exports

import akka.NotUsed
import akka.stream.scaladsl.Source
import akka.util.ByteString
import controllers.application.exports.CsvFileStreaming.makeFileName
import drt.shared.CrunchApi.MillisSinceEpoch
import org.slf4j.{Logger, LoggerFactory}
import play.api.http.Status.OK
import play.api.http.{HttpChunk, HttpEntity, Writeable}
import play.api.mvc.{ResponseHeader, Result, Results}
import play.mvc.StaticFileMimeTypes.fileMimeTypes
import uk.gov.homeoffice.drt.arrivals.{ApiFlightWithSplits, Splits}
import uk.gov.homeoffice.drt.ports.SplitRatiosNs
import uk.gov.homeoffice.drt.time.SDate
import uk.gov.homeoffice.drt.ports.{PortCode, SplitRatiosNs}
import uk.gov.homeoffice.drt.ports.Terminals.Terminal
import uk.gov.homeoffice.drt.time.{LocalDate, SDate}
import uk.gov.homeoffice.drt.time.TimeZoneHelper.europeLondonTimeZone

import scala.compat.java8.OptionConverters.RichOptionalGeneric


object Exports {
val log: Logger = LoggerFactory.getLogger(getClass)
Expand All @@ -22,4 +33,26 @@ object Exports {
s.splits.map(s => (s"API Actual - ${s.paxTypeAndQueue.displayName}", s.paxCount))
}
.flatten

def streamExport(portCode: PortCode,
terminals: Seq[Terminal],
start: LocalDate,
end: LocalDate,
stream: Source[String, NotUsed],
exportName: String): Result = {
implicit val writeable: Writeable[String] = Writeable(ByteString.fromString, Option("text/csv"))

val header = ResponseHeader(OK)
val disableNginxProxyBuffering = "X-Accel-Buffering" -> "no"
val fileName = makeFileName(exportName, terminals, start, end, portCode) + ".csv"

Result(
header = header.copy(headers = header.headers ++ Results.contentDispositionHeader(inline = true, Option(fileName)) ++ Option(disableNginxProxyBuffering)),
body = HttpEntity.Chunked(
stream.map(c => HttpChunk.Chunk(writeable.transform(c))),
fileMimeTypes.forFileName(fileName).asScala
)
)
}

}
5 changes: 1 addition & 4 deletions server/src/main/scala/services/exports/flights/templates/FlightsExport.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@ trait FlightsExport {

def csvStream(flightsStream: Source[(Iterable[ApiFlightWithSplits], VoyageManifests), NotUsed]): Source[String, NotUsed] =
filterAndSort(flightsStream)
.map { case (fws, maybeManifest) =>
flightToCsvRow(fws, maybeManifest) + "\n"
}
.map { case (fws, maybeManifest) => flightToCsvRow(fws, maybeManifest) + "\n" }
.prepend(Source(List(headings + "\n")))

private def filterAndSort(flightsStream: Source[(Iterable[ApiFlightWithSplits], VoyageManifests), NotUsed],
Expand All @@ -51,5 +49,4 @@ trait FlightsExport {
(fws, maybeManifest)
}
}

}

0 comments on commit 380a1f7

Please sign in to comment.