Skip to content

Commit

Permalink
Merge pull request #1962 from UKHomeOffice/DRTII-1693-Multidate-expor…
Browse files Browse the repository at this point in the history
…t-for-BRS-not-returning-the-correct-data

Drtii 1693 multidate export for brs not returning the correct data
  • Loading branch information
richbirch authored Nov 29, 2024
2 parents 198ce19 + 2d5df6b commit c60c1dd
Show file tree
Hide file tree
Showing 24 changed files with 58 additions and 78 deletions.
6 changes: 3 additions & 3 deletions server/src/main/scala/actors/FlightLookups.scala
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import uk.gov.homeoffice.drt.ports.Queues.Queue
import uk.gov.homeoffice.drt.ports.Terminals.Terminal
import uk.gov.homeoffice.drt.time.{SDateLike, UtcDate}

import scala.concurrent.ExecutionContext
import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration._
import scala.language.postfixOps

Expand All @@ -31,7 +31,7 @@ trait FlightLookupsLike {
val terminalSplits: Terminal => Option[Splits]

def updateFlights(removalMessageCutOff: Option[FiniteDuration],
updateLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Unit,
updateLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Future[Unit],
)
(requestHistoricSplitsActor: Option[ActorRef],
requestHistoricPaxActor: Option[ActorRef],
Expand Down Expand Up @@ -66,7 +66,7 @@ case class FlightLookups(system: ActorSystem,
removalMessageCutOff: Option[FiniteDuration],
paxFeedSourceOrder: List[FeedSource],
terminalSplits: Terminal => Option[Splits],
updateLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Unit,
updateLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Future[Unit],
) extends FlightLookupsLike {
override val requestAndTerminateActor: ActorRef = system.actorOf(Props(new RequestAndTerminateActor()), "flights-lookup-kill-actor")

Expand Down
4 changes: 2 additions & 2 deletions server/src/main/scala/actors/MinuteLookups.scala
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ trait MinuteLookupsLike {
requestAndTerminateActor.ask(RequestAndTerminate(actor, container)).mapTo[Set[TerminalUpdateRequest]]
}

def updateCrunchMinutes(updateLiveView: (UtcDate, Iterable[CrunchMinute]) => Unit): ((Terminal, UtcDate), MinutesContainer[CrunchMinute, TQM]) => Future[Set[TerminalUpdateRequest]] =
def updateCrunchMinutes(updateLiveView: (UtcDate, Iterable[CrunchMinute]) => Future[Unit]): ((Terminal, UtcDate), MinutesContainer[CrunchMinute, TQM]) => Future[Set[TerminalUpdateRequest]] =
(terminalDate: (Terminal, UtcDate), container: MinutesContainer[CrunchMinute, TQM]) => {
val (terminal, date) = terminalDate
val actor = system.actorOf(TerminalDayQueuesActor.props(Option(updateLiveView))(terminal, date, now))
Expand Down Expand Up @@ -81,7 +81,7 @@ trait MinuteLookupsLike {
case class MinuteLookups(now: () => SDateLike,
expireAfterMillis: Int,
queuesByTerminal: Map[Terminal, Seq[Queue]],
updateLiveView: (UtcDate, Iterable[CrunchMinute]) => Unit,
updateLiveView: (UtcDate, Iterable[CrunchMinute]) => Future[Unit],
)
(implicit val ec: ExecutionContext, val system: ActorSystem) extends MinuteLookupsLike {
override val requestAndTerminateActor: ActorRef = system.actorOf(Props(new RequestAndTerminateActor()), "minutes-lookup-kill-actor")
Expand Down
15 changes: 8 additions & 7 deletions server/src/main/scala/actors/daily/TerminalDayFlightActor.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import uk.gov.homeoffice.drt.protobuf.serialisation.FlightMessageConversion
import uk.gov.homeoffice.drt.protobuf.serialisation.FlightMessageConversion._
import uk.gov.homeoffice.drt.time.{SDate, SDateLike, UtcDate}

import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration

object TerminalDayFlightActor {
Expand All @@ -32,7 +33,7 @@ object TerminalDayFlightActor {
terminalSplits: Option[Splits],
requestHistoricSplitsActor: Option[ActorRef],
requestHistoricPaxActor: Option[ActorRef],
maybeUpdateLiveView: Option[(Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Unit],
maybeUpdateLiveView: Option[(Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Future[Unit]],
): Props =
Props(new TerminalDayFlightActor(
year = date.year,
Expand Down Expand Up @@ -84,7 +85,7 @@ class TerminalDayFlightActor(year: Int,
terminalSplits: Option[Splits],
maybeRequestHistoricSplitsActor: Option[ActorRef],
maybeRequestHistoricPaxActor: Option[ActorRef],
maybeUpdateLiveView: Option[(Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Unit],
maybeUpdateLiveView: Option[(Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Future[Unit]],
) extends RecoveryActorLike {
val loggerSuffix: String = maybePointInTime match {
case None => ""
Expand Down Expand Up @@ -200,8 +201,8 @@ class TerminalDayFlightActor(year: Int,
acceptableExistingSources.isEmpty
}

private def applyDiffAndPersist(applyDiff: (FlightsWithSplits, Long, List[FeedSource]) => (FlightsWithSplits, Set[Long], Iterable[ApiFlightWithSplits], Iterable[UniqueArrival])
): Set[TerminalUpdateRequest] = {
private def applyDiffAndRequestMissingData(applyDiff: (FlightsWithSplits, Long, List[FeedSource]) => (FlightsWithSplits, Set[Long], Iterable[ApiFlightWithSplits], Iterable[UniqueArrival])
): Set[TerminalUpdateRequest] = {
val (updatedState, minutesToUpdate, updates, removals) = applyDiff(state, now().millisSinceEpoch, paxFeedSourceOrder)

state = updatedState
Expand All @@ -218,23 +219,23 @@ class TerminalDayFlightActor(year: Int,

private def updateAndPersistDiffAndAck(diff: FlightsWithSplitsDiff): Unit =
if (diff.nonEmpty) {
val updateRequests = applyDiffAndPersist(diff.applyTo)
val updateRequests = applyDiffAndRequestMissingData(diff.applyTo)
val message = flightWithSplitsDiffToMessage(diff, now().millisSinceEpoch)
persistAndMaybeSnapshotWithAck(message, List((sender(), updateRequests)))
}
else sender() ! Set.empty

private def updateAndPersistDiffAndAck(diff: ArrivalsDiff): Unit =
if (diff.toUpdate.nonEmpty || diff.toRemove.nonEmpty) {
val updateRequests = applyDiffAndPersist(diff.applyTo)
val updateRequests = applyDiffAndRequestMissingData(diff.applyTo)
val message = arrivalsDiffToMessage(diff, now().millisSinceEpoch)
persistAndMaybeSnapshotWithAck(message, List((sender(), updateRequests)))
} else sender() ! Set.empty

private def updateAndPersistDiffAndAck(diff: SplitsForArrivals): Unit =
if (diff.splits.nonEmpty) {
val timestamp = now().millisSinceEpoch
val updateRequests = applyDiffAndPersist(diff.applyTo)
val updateRequests = applyDiffAndRequestMissingData(diff.applyTo)
val message = splitsForArrivalsToMessage(diff, timestamp)
persistAndMaybeSnapshotWithAck(message, List((sender(), updateRequests)))
} else sender() ! Set.empty
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import uk.gov.homeoffice.drt.time.TimeZoneHelper.utcTimeZone
import uk.gov.homeoffice.drt.time.{SDate, SDateLike, UtcDate}

import scala.collection.mutable
import scala.concurrent.Future


abstract class TerminalDayLikeActor[VAL <: MinuteLike[VAL, INDEX], INDEX <: WithTimeAccessor, M <: GeneratedMessage](year: Int,
Expand All @@ -23,6 +24,8 @@ abstract class TerminalDayLikeActor[VAL <: MinuteLike[VAL, INDEX], INDEX <: With
now: () => SDateLike,
override val maybePointInTime: Option[MillisSinceEpoch],
) extends RecoveryActorLike {
implicit val ec = context.dispatcher

val loggerSuffix: String = maybePointInTime match {
case None => ""
case Some(pit) => f"@${SDate(pit).toISOString}"
Expand All @@ -33,7 +36,7 @@ abstract class TerminalDayLikeActor[VAL <: MinuteLike[VAL, INDEX], INDEX <: With

val state: mutable.Map[INDEX, VAL] = mutable.Map[INDEX, VAL]()

val onUpdate: Option[(UtcDate, Iterable[VAL]) => Unit] = None
val onUpdate: Option[(UtcDate, Iterable[VAL]) => Future[Unit]] = None

override def persistenceId: String = f"terminal-$persistenceIdType-${terminal.toString.toLowerCase}-$year-$month%02d-$day%02d"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import uk.gov.homeoffice.drt.ports.Terminals.Terminal
import uk.gov.homeoffice.drt.protobuf.messages.CrunchState.{CrunchMinuteMessage, CrunchMinutesMessage}
import uk.gov.homeoffice.drt.time.{SDateLike, UtcDate}

import scala.concurrent.Future


object TerminalDayQueuesActor {
def props(maybeUpdateLiveView: Option[(UtcDate, Iterable[CrunchMinute]) => Unit])
def props(maybeUpdateLiveView: Option[(UtcDate, Iterable[CrunchMinute]) => Future[Unit]])
(terminal: Terminal,
date: UtcDate,
now: () => SDateLike,
Expand All @@ -28,7 +30,7 @@ class TerminalDayQueuesActor(year: Int,
terminal: Terminal,
val now: () => SDateLike,
maybePointInTime: Option[MillisSinceEpoch],
override val onUpdate: Option[(UtcDate, Iterable[CrunchMinute]) => Unit],
override val onUpdate: Option[(UtcDate, Iterable[CrunchMinute]) => Future[Unit]],
) extends
TerminalDayLikeActor[CrunchMinute, TQM, CrunchMinuteMessage](year, month, day, terminal, now, maybePointInTime) {
override val persistenceIdType: String = "queues"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ import play.api.mvc._
import services.api.v1.FlightExport
import services.api.v1.serialisation.FlightApiJsonProtocol
import spray.json.enrichAny
import uk.gov.homeoffice.drt.arrivals.{Arrival, FlightsWithSplits}
import uk.gov.homeoffice.drt.auth.Roles.{ApiFlightAccess, ApiQueueAccess, SuperAdmin}
import uk.gov.homeoffice.drt.arrivals.{ApiFlightWithSplits, Arrival, FlightsWithSplits}
import uk.gov.homeoffice.drt.auth.Roles.{ApiFlightAccess, SuperAdmin}
import uk.gov.homeoffice.drt.crunchsystem.DrtSystemInterface
import uk.gov.homeoffice.drt.ports.FeedSource
import uk.gov.homeoffice.drt.ports.Terminals.Terminal
Expand Down Expand Up @@ -72,17 +72,19 @@ class FlightsApiController @Inject()(cc: ControllerComponents, ctrl: DrtSystemIn
}
Source(DateRange(startDate, endDate))
.mapAsync(1) { date =>
ctrl.applicationService.flightsProvider.allTerminalsScheduledOn(date).runForeach { flights =>
ctrl.updateFlightsLiveView(flights, Seq.empty)
log.info(s"Updated flights for $date")
}
log.info(s"Populating flights for $date")
ctrl.applicationService.flightsProvider.allTerminalsScheduledOn(date)
.runWith(Sink.fold(Seq.empty[ApiFlightWithSplits])(_ ++ _))
.flatMap { flights =>
ctrl.updateFlightsLiveView(flights, Seq.empty)
.map(_ => log.info(s"Updated flights for $date"))
}
}
.runWith(Sink.ignore)
Ok("Flights populating")
}
}


private def parseOptionalEndDate(maybeString: Option[String], default: SDateLike): SDateLike =
maybeString match {
case None => default
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import controllers.application.AuthController
import drt.shared.CrunchApi
import drt.shared.CrunchApi.MinutesContainer
import play.api.mvc._
import providers.MinutesProvider
import services.api.v1.QueueExport
import services.api.v1.serialisation.QueueApiJsonProtocol
import spray.json.enrichAny
Expand Down
5 changes: 2 additions & 3 deletions server/src/main/scala/providers/FlightsProvider.scala
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,8 @@ case class FlightsProvider(flightsRouterActor: ActorRef)
val start = SDate(date)
val end = start.addDays(1).addMinutes(-1)
val request = PartitionedPortStateActor.GetFlights(start.millisSinceEpoch, end.millisSinceEpoch)
flightsByUtcDate(request).map { case (_, flights) =>
flights.filter(f => SDate(f.apiFlight.Scheduled).toUtcDate == date)
}
flightsByUtcDate(request)
.map { case (_, flights) => flights.filter(f => SDate(f.apiFlight.Scheduled).toUtcDate == date) }
}

def allTerminalsDateRangeScheduledOrPcp: (UtcDate, UtcDate) => Source[(UtcDate, Seq[ApiFlightWithSplits]), NotUsed] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import uk.gov.homeoffice.drt.routes.UserRoleProviderLike
import uk.gov.homeoffice.drt.service.{ApplicationService, FeedService}
import uk.gov.homeoffice.drt.time._

import scala.concurrent.ExecutionContext
import scala.concurrent.{ExecutionContext, Future}

trait DrtSystemInterface extends UserRoleProviderLike
with FeatureGuideProviderLike
Expand Down Expand Up @@ -72,19 +72,16 @@ trait DrtSystemInterface extends UserRoleProviderLike
lazy val flightsForPcpDateRange: (LocalDate, LocalDate, Seq[Terminal]) => Source[(UtcDate, Seq[ApiFlightWithSplits]), NotUsed] =
flightDao.flightsForPcpDateRange(airportConfig.portCode, paxFeedSourceOrder, aggregatedDb.run)

lazy val updateFlightsLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Unit = {
lazy val updateFlightsLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Future[Unit] = {
val doUpdate = FlightsLiveView.updateFlightsLiveView(flightDao, aggregatedDb, airportConfig.portCode)
(updates, removals) =>
doUpdate(updates, removals)
}

lazy val update15MinuteQueueSlotsLiveView: (UtcDate, Iterable[CrunchMinute]) => Unit = {
lazy val update15MinuteQueueSlotsLiveView: (UtcDate, Iterable[CrunchMinute]) => Future[Unit] = {
val doUpdate = QueuesLiveView.updateQueuesLiveView(queueSlotDao, aggregatedDb, airportConfig.portCode)
(date, updates) =>
doUpdate(date, updates)
.recover { case e: Throwable =>
log.error(s"Error updating FlightsLiveView: ${e.getMessage}")
}
doUpdate(date, updates).map(_ => ())
}

def getRoles(config: Configuration, headers: Headers, session: Session): Set[Role] = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ object TestActors {
day: Int,
terminal: Terminal,
now: () => SDateLike,
onUpdate: Option[(UtcDate, Iterable[CrunchMinute]) => Unit],
onUpdate: Option[(UtcDate, Iterable[CrunchMinute]) => Future[Unit]],
) extends TerminalDayQueuesActor(year, month, day, terminal, now, None, onUpdate) with Resettable {
override def resetState(): Unit = state.clear()

Expand All @@ -409,7 +409,7 @@ object TestActors {
paxFeedSourceOrder: List[FeedSource],
requestHistoricSplitsActor: Option[ActorRef],
requestHistoricPaxActor: Option[ActorRef],
maybeUpdateLiveView: Option[(Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Unit],
maybeUpdateLiveView: Option[(Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Future[Unit]],
)
extends TerminalDayFlightActor(
year,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import play.api.mvc.{Headers, Session}
import slickdb._
import uk.gov.homeoffice.drt.auth.Roles.Role
import uk.gov.homeoffice.drt.crunchsystem.{ActorsServiceLike, DrtSystemInterface}
import uk.gov.homeoffice.drt.db.dao.{FlightDao, IABFeatureDao, IUserFeedbackDao}
import uk.gov.homeoffice.drt.ports.{AirportConfig, PortCode}
import uk.gov.homeoffice.drt.db.dao.{IABFeatureDao, IUserFeedbackDao}
import uk.gov.homeoffice.drt.ports.AirportConfig
import uk.gov.homeoffice.drt.service.FeedService
import uk.gov.homeoffice.drt.testsystem.RestartActor.StartTestSystem
import uk.gov.homeoffice.drt.testsystem.crunchsystem.TestPersistentStateActors
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ case class TestFlightLookups(system: ActorSystem,
queuesByTerminal: Map[Terminal, Seq[Queue]],
paxFeedSourceOrder: List[FeedSource],
terminalSplits: Terminal => Option[Splits],
updateLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Unit,
updateLiveView: (Iterable[ApiFlightWithSplits], Iterable[UniqueArrival]) => Future[Unit],
) extends FlightLookupsLike {
override val requestAndTerminateActor: ActorRef = system.actorOf(Props(new RequestAndTerminateActor()), "test-flights-lookup-kill-actor")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ case class TestMinuteLookups(system: ActorSystem,
now: () => SDateLike,
expireAfterMillis: Int,
queuesByTerminal: Map[Terminal, Seq[Queue]],
updateLiveView: (UtcDate, Iterable[CrunchMinute]) => Unit,
updateLiveView: (UtcDate, Iterable[CrunchMinute]) => Future[Unit],
)
(implicit val ec: ExecutionContext) extends MinuteLookupsLike {
override val requestAndTerminateActor: ActorRef = system.actorOf(Props(new RequestAndTerminateActor()), "test-minutes-lookup-kill-actor")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ class FlightsRouterActorSpec extends CrunchTestLike {

"Concerning persistence of flights" >> {
"Given a router, I should see updates sent to it are persisted" >> {
val lookups = FlightLookups(system, myNow, queuesByTerminal = Map(T1 -> Seq(EeaDesk, NonEeaDesk, EGate)), None, paxFeedSourceOrder, _ => None, (_, _) => ())
val lookups = FlightLookups(system, myNow, queuesByTerminal = Map(T1 -> Seq(EeaDesk, NonEeaDesk, EGate)), None, paxFeedSourceOrder, _ => None, (_, _) => Future.successful(()))
val router = lookups.flightsRouterActor

val scheduled = "2021-06-01T00:00"
Expand Down Expand Up @@ -315,7 +315,7 @@ class FlightsRouterActorSpec extends CrunchTestLike {

"Add red list pax to an existing arrival" in {
val redListNow = SDate("2021-06-24T12:10:00")
val lookups = FlightLookups(system, () => redListNow, Map(T1 -> Seq(), T2 -> Seq()), None, paxFeedSourceOrder, _ => None, (_, _) => ())
val lookups = FlightLookups(system, () => redListNow, Map(T1 -> Seq(), T2 -> Seq()), None, paxFeedSourceOrder, _ => None, (_, _) => Future.successful(()))
val redListPassengers = RedListPassengers("BA0001", PortCode("LHR"), SDate(scheduled), redListPax)
val neboArrivalActor: ActorRef = system.actorOf(NeboArrivalActor.props(redListPassengers, () => redListNow))
val arrival = ArrivalGenerator.live(iata = "BA0001", terminal = T1, schDt = scheduled).toArrival(LiveFeedSource)
Expand All @@ -335,7 +335,7 @@ class FlightsRouterActorSpec extends CrunchTestLike {

"Add red list pax counts to the appropriate arrivals" in {
val redListNow = SDate("2021-06-24T12:10:00")
val lookups = FlightLookups(system, () => redListNow, Map(T1 -> Seq(), T2 -> Seq()), None, paxFeedSourceOrder, _ => None, (_, _) => ())
val lookups = FlightLookups(system, () => redListNow, Map(T1 -> Seq(), T2 -> Seq()), None, paxFeedSourceOrder, _ => None, (_, _) => Future.successful(()))
val flightsRouter = lookups.flightsRouterActor
val scheduled2 = "2021-06-24T15:05"
val arrivalT1 = ArrivalGenerator.live(iata = "BA0001", terminal = T1, schDt = scheduled).toArrival(LiveFeedSource)
Expand Down Expand Up @@ -365,7 +365,7 @@ class FlightsRouterActorSpec extends CrunchTestLike {

"Retain red list pax counts after updating an arrival" in {
val redListNow = SDate("2021-06-24T12:10:00")
val lookups = FlightLookups(system, () => redListNow, Map(T1 -> Seq(), T2 -> Seq()), None, paxFeedSourceOrder, _ => None, (_, _) => ())
val lookups = FlightLookups(system, () => redListNow, Map(T1 -> Seq(), T2 -> Seq()), None, paxFeedSourceOrder, _ => None, (_, _) => Future.successful(()))
val flightsRouter = lookups.flightsRouterActor
val arrivalT1 = ArrivalGenerator.live(iata = "BA0001", terminal = T1, schDt = scheduled).toArrival(LiveFeedSource)
val redListPax = util.RandomString.getNRandomString(10, 10)
Expand Down
Loading

0 comments on commit c60c1dd

Please sign in to comment.