Add required information to DBP NA<->FE API for removal timeline supp…

…ort (#3268)

Task/Issue URL:
https://app.asana.com/0/1203581873609357/1208164179397484/f
Tech Design URL:
Initial API additions:
https://app.asana.com/0/481882893211075/1208203761791777/f
Parent - child api info and matching logic:
https://app.asana.com/0/72649045549333/1208303545057273/f
Where to put the matching logic:
https://app.asana.com/0/481882893211075/1208325118544624/f
CC:

**Description**:
- Adds four new fields as per the first tech design. Also moves some of
the construction of these objects to constructors.
- Also removes an old date field that isn't officially in the api, and
apparently varied what it was used for. It seemed unused on the native
side, and the FE side doesn't know it exist.

- It then also adds "parent child" matching logic so we can identify
orphaned child records that will never be removed.

**Steps to test this PR**:
1. In DataBrokerProtectionWebUIURLSettings, change the production URL to
"https://use-devtesting27.duckduckgo.com/dbp", Alex made an FE build we
can use to test
2. In the DBP UI, click on individual records, this will show you things
like the opt out submitted date
3. Make sure these values make sense, e.g. we don't ever show 1970, the
timeline follows linear time, nothing violates causality etc.

<!--
Tagging instructions
If this PR isn't ready to be merged for whatever reason it should be
marked with the `DO NOT MERGE` label (particularly if it's a draft)
If it's pending Product Review/PFR, please add the `Pending Product
Review` label.

If at any point it isn't actively being worked on/ready for
review/otherwise moving forward (besides the above PR/PFR exception)
strongly consider closing it (or not opening it in the first place). If
you decide not to close it, make sure it's labelled to make it clear the
PRs state and comment with more information.
-->

**Definition of Done**:

* [ ] Does this PR satisfy our [Definition of
Done](https://app.asana.com/0/1202500774821704/1207634633537039/f)?

---
###### Internal references:
[Pull Request Review
Checklist](https://app.asana.com/0/1202500774821704/1203764234894239/f)
[Software Engineering
Expectations](https://app.asana.com/0/59792373528535/199064865822552)
[Technical Design
Template](https://app.asana.com/0/59792373528535/184709971311943)
[Pull Request
Documentation](https://app.asana.com/0/1202500774821704/1204012835277482/f)

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDataManager.swift

@@ -326,10 +326,10 @@ extension InMemoryDataCache: DBPUICommunicationDelegate {
         // 2. We map the brokers to the UI model
             .flatMap { dataBroker -> [DBPUIDataBroker] in
                 var result: [DBPUIDataBroker] = []
-                result.append(DBPUIDataBroker(name: dataBroker.name, url: dataBroker.url))
+                result.append(DBPUIDataBroker(name: dataBroker.name, url: dataBroker.url, parentURL: dataBroker.parent))
 
                 for mirrorSite in dataBroker.mirrorSites {
-                    result.append(DBPUIDataBroker(name: mirrorSite.name, url: mirrorSite.url))
+                    result.append(DBPUIDataBroker(name: mirrorSite.name, url: mirrorSite.url, parentURL: dataBroker.parent))
                 }
                 return result
             }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DBPUICommunicationModel.swift

@@ -76,6 +76,15 @@ struct DBPUIUserProfileAddress: Codable {
     let zipCode: String?
 }
 
+extension DBPUIUserProfileAddress {
+    init(addressCityState: AddressCityState) {
+        self.init(street: addressCityState.fullAddress,
+                  city: addressCityState.city,
+                  state: addressCityState.state,
+                  zipCode: nil)
+    }
+}
+
 /// Message Object representing a user profile containing one or more names and addresses
 /// also contains the user profile's birth year
 struct DBPUIUserProfile: Codable {
@@ -105,11 +114,13 @@ struct DBPUIDataBroker: Codable, Hashable {
     let name: String
     let url: String
     let date: Double?
+    let parentURL: String?
 
-    init(name: String, url: String, date: Double? = nil) {
+    init(name: String, url: String, date: Double? = nil, parentURL: String?) {
         self.name = name
         self.url = url
         self.date = date
+        self.parentURL = parentURL
     }
 
     func hash(into hasher: inout Hasher) {
@@ -135,7 +146,72 @@ struct DBPUIDataBrokerProfileMatch: Codable {
     let addresses: [DBPUIUserProfileAddress]
     let alternativeNames: [String]
     let relatives: [String]
-    let date: Double? // Used in some methods to set the removedDate or found date
+    let foundDate: Double
+    let optOutSubmittedDate: Double?
+    let estimatedRemovalDate: Double?
+    let removedDate: Double?
+    let hasMatchingRecordOnParentBroker: Bool
+}
+
+extension DBPUIDataBrokerProfileMatch {
+    init(optOutJobData: OptOutJobData,
+         dataBrokerName: String,
+         dataBrokerURL: String,
+         dataBrokerParentURL: String?,
+         parentBrokerOptOutJobData: [OptOutJobData]?) {
+        let extractedProfile = optOutJobData.extractedProfile
+
+        /*
+         createdDate used to not exist in the DB, so in the migration we defaulted it to Unix Epoch zero (i.e. 1970)
+         If that's the case, we should rely on the events instead
+         We don't do that all the time since it's unnecssarily expensive trawling through events, and
+         this is involved in some already heavy endpoints
+
+         optOutSubmittedDate also used to not exist, but instead defaults to nil
+         However, it could be nil simply because the opt out hasn't been submitted yet. So since we don't want to
+         look through events unneccesarily, we instead only look for it if the createdDate is 1970
+         */
+        var foundDate = optOutJobData.createdDate
+        var optOutSubmittedDate = optOutJobData.submittedSuccessfullyDate
+        if foundDate == Date(timeIntervalSince1970: 0) {
+            let foundEvents = optOutJobData.historyEvents.filter { $0.isMatchesFoundEvent() }
+            let firstFoundEvent = foundEvents.min(by: { $0.date < $1.date })
+            if let firstFoundEventDate = firstFoundEvent?.date {
+                foundDate = firstFoundEventDate
+            } else {
+                assertionFailure("No matching MatchFound event for an extract profile found")
+            }
+
+            let optOutSubmittedEvents = optOutJobData.historyEvents.filter { $0.type == .optOutRequested }
+            let firstOptOutEvent = optOutSubmittedEvents.min(by: { $0.date < $1.date })
+            optOutSubmittedDate = firstOptOutEvent?.date
+        }
+        let estimatedRemovalDate = Calendar.current.date(byAdding: .day, value: 14, to: optOutSubmittedDate ?? foundDate)
+
+        // Check for any matching records on the parent broker
+        let hasFoundParentMatch = parentBrokerOptOutJobData?.contains { parentOptOut in
+            extractedProfile.doesMatchExtractedProfile(parentOptOut.extractedProfile)
+        } ?? false
+
+        self.init(dataBroker: DBPUIDataBroker(name: dataBrokerName, url: dataBrokerURL, parentURL: dataBrokerParentURL),
+                  name: extractedProfile.fullName ?? "No name",
+                  addresses: extractedProfile.addresses?.map {DBPUIUserProfileAddress(addressCityState: $0) } ?? [],
+                  alternativeNames: extractedProfile.alternativeNames ?? [String](),
+                  relatives: extractedProfile.relatives ?? [String](),
+                  foundDate: foundDate.timeIntervalSince1970,
+                  optOutSubmittedDate: optOutSubmittedDate?.timeIntervalSince1970,
+                  estimatedRemovalDate: estimatedRemovalDate?.timeIntervalSince1970,
+                  removedDate: extractedProfile.removedDate?.timeIntervalSince1970,
+                  hasMatchingRecordOnParentBroker: hasFoundParentMatch)
+    }
+
+    init(optOutJobData: OptOutJobData, dataBroker: DataBroker, parentBrokerOptOutJobData: [OptOutJobData]?) {
+        self.init(optOutJobData: optOutJobData,
+                  dataBrokerName: dataBroker.name,
+                  dataBrokerURL: dataBroker.url,
+                  dataBrokerParentURL: dataBroker.parent,
+                  parentBrokerOptOutJobData: parentBrokerOptOutJobData)
+    }
 }
 
 /// Protocol to represent a message that can be passed from the host to the UI
@@ -156,6 +232,27 @@ struct DBPUIOptOutMatch: DBPUISendableMessage {
     let alternativeNames: [String]
     let addresses: [DBPUIUserProfileAddress]
     let date: Double
+    let foundDate: Double
+    let optOutSubmittedDate: Double?
+    let estimatedRemovalDate: Double?
+    let removedDate: Double?
+}
+
+extension DBPUIOptOutMatch {
+    init?(profileMatch: DBPUIDataBrokerProfileMatch, matches: Int) {
+        guard let removedDate = profileMatch.removedDate else { return nil }
+        let dataBroker = profileMatch.dataBroker
+        self.init(dataBroker: dataBroker,
+                  matches: matches,
+                  name: profileMatch.name,
+                  alternativeNames: profileMatch.alternativeNames,
+                  addresses: profileMatch.addresses,
+                  date: removedDate,
+                  foundDate: profileMatch.foundDate,
+                  optOutSubmittedDate: profileMatch.optOutSubmittedDate,
+                  estimatedRemovalDate: profileMatch.estimatedRemovalDate,
+                  removedDate: removedDate)
+    }
 }
 
 /// Data representing the initial scan progress

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ExtractedProfile.swift

@@ -56,7 +56,7 @@ struct ExtractProfileSelectors: Codable, Sendable {
     }
 }
 
-struct AddressCityState: Codable {
+struct AddressCityState: Codable, Hashable {
     let city: String
     let state: String
 
@@ -166,10 +166,51 @@ struct ExtractedProfile: Codable, Sendable {
             identifier: self.identifier
         )
     }
+
+    /*
+     Matching records are:
+     1/ Completely identical records (same name, addresses, ages, etc)
+     2/ Records that overlap completely (record A has all the data of record B, but might have
+        extra information as well (e.g. an extra address, a middle name where record B doesn't)
+        I.e. B is a subset of A, or vice versa
+     However, we ignore some of the properties
+     So, basically age == age, we ignore phone numbers and email, and then everything else one should be a subset of the other
+     */
+    func doesMatchExtractedProfile(_ extractedProfile: ExtractedProfile) -> Bool {
+        if age != extractedProfile.age {
+            return false
+        }
+
+        if name != extractedProfile.name {
+            return false
+        }
+
+        if !(alternativeNames ?? []).isASubSetOrSuperSetOf(extractedProfile.alternativeNames ?? []) {
+            return false
+        }
+
+        if !(addresses ?? []).isASubSetOrSuperSetOf(extractedProfile.addresses ?? []) {
+            return false
+        }
+
+        if !(relatives ?? []).isASubSetOrSuperSetOf(extractedProfile.relatives ?? []) {
+            return false
+        }
+
+        return true
+    }
 }
 
 extension ExtractedProfile: Equatable {
     static func == (lhs: ExtractedProfile, rhs: ExtractedProfile) -> Bool {
         lhs.name == rhs.name
     }
 }
+
+private extension Sequence where Element: Hashable {
+    func isASubSetOrSuperSetOf<Settable>(_ sequence: Settable) -> Bool where Settable: Sequence, Element == Settable.Element {
+        let setA = Set(self)
+        let setB = Set(sequence)
+        return setA.isSubset(of: setB) || setB.isSubset(of: setA)
+    }
+}

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/HistoryEvent.swift

@@ -69,4 +69,13 @@ public struct HistoryEvent: Identifiable, Sendable {
             return false
         }
     }
+
+    func isMatchesFoundEvent() -> Bool {
+        switch type {
+        case .matchesFound:
+            return true
+        default:
+            return false
+        }
+    }
 }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/DBPUICommunicationLayer.swift

@@ -78,7 +78,7 @@ struct DBPUICommunicationLayer: Subfeature {
     weak var delegate: DBPUICommunicationDelegate?
 
     private enum Constants {
-        static let version = 5
+        static let version = 6
     }
 
     internal init(webURLSettings: DataBrokerProtectionWebUIURLSettingsRepresentable,