Add orphaned profiles pixel (#3340)

Task/Issue URL:
https://app.asana.com/0/481882893211075/1208361130560688/f
Tech Design URL:
CC:

**Description**:
Adds a pixel to help us know if our "orphaned profile detection" is
working.

**Steps to test this PR**:
1. Having completed a scan in the last week, and making sure you're
connected to the background agent with the debugger, you can turn pixel
logging on, and in DataBrokerProtectionEventPixels line 68, comment out
the check for if we should send weekly pixels so that it always sends
them
2. Otherwise scrutinize the unit tests for missing test cases

<!--
Tagging instructions
If this PR isn't ready to be merged for whatever reason it should be
marked with the `DO NOT MERGE` label (particularly if it's a draft)
If it's pending Product Review/PFR, please add the `Pending Product
Review` label.

If at any point it isn't actively being worked on/ready for
review/otherwise moving forward (besides the above PR/PFR exception)
strongly consider closing it (or not opening it in the first place). If
you decide not to close it, make sure it's labelled to make it clear the
PRs state and comment with more information.
-->

**Definition of Done**:

* [ ] Does this PR satisfy our [Definition of
Done](https://app.asana.com/0/1202500774821704/1207634633537039/f)?

---
###### Internal references:
[Pull Request Review
Checklist](https://app.asana.com/0/1202500774821704/1203764234894239/f)
[Software Engineering
Expectations](https://app.asana.com/0/59792373528535/199064865822552)
[Technical Design
Template](https://app.asana.com/0/59792373528535/184709971311943)
[Pull Request
Documentation](https://app.asana.com/0/1202500774821704/1204012835277482/f)

---------

Co-authored-by: Pete Smith <[email protected]>

DuckDuckGo/DBP/DataBrokerProtectionPixelsHandler.swift

@@ -110,7 +110,8 @@ public class DataBrokerProtectionPixelsHandler: EventMapping<DataBrokerProtectio
                     .invalidPayload,
                     .pixelTest,
                     .customDataBrokerStatsOptoutSubmit,
-                    .customGlobalStatsOptoutSubmit:
+                    .customGlobalStatsOptoutSubmit,
+                    .weeklyChildBrokerOrphanedOptOuts:
 
                 PixelKit.fire(event)
 

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionEventPixels.swift

@@ -130,6 +130,8 @@ final class DataBrokerProtectionEventPixels {
 
         handler.fire(.weeklyReportScanning(hadNewMatch: newMatchesFoundInTheLastWeek > 0, hadReAppereance: reAppereancesInTheLastWeek > 0, scanCoverage: percentageOfBrokersScanned.toString))
         handler.fire(.weeklyReportRemovals(removals: removalsInTheLastWeek))
+
+        fireWeeklyChildBrokerOrphanedOptOutsPixels(for: data)
     }
 
     private func hadScanThisWeek(_ brokerProfileQuery: BrokerProfileQueryData) -> Bool {
@@ -149,6 +151,77 @@ final class DataBrokerProtectionEventPixels {
     }
 }
 
+// MARK: - Orphaned profiles stuff
+
+extension DataBrokerProtectionEventPixels {
+
+    func weeklyOptOuts(for brokerProfileQueries: [BrokerProfileQueryData]) -> [OptOutJobData] {
+        let optOuts = brokerProfileQueries.flatMap { $0.optOutJobData }
+        let weeklyOptOuts = optOuts.filter { !didWeekPassedBetweenDates(start: $0.createdDate, end: Date()) }
+        return weeklyOptOuts
+    }
+
+    func fireWeeklyChildBrokerOrphanedOptOutsPixels(for data: [BrokerProfileQueryData]) {
+        let brokerURLsToQueryData = Dictionary(grouping: data, by: { $0.dataBroker.url })
+        let childBrokerURLsToOrphanedProfilesCount = childBrokerURLsToOrphanedProfilesWeeklyCount(for: data)
+        for (key, value) in childBrokerURLsToOrphanedProfilesCount {
+            guard let childQueryData = brokerURLsToQueryData[key],
+                  let childBrokerName = childQueryData.first?.dataBroker.name,
+                  let parentURL = childQueryData.first?.dataBroker.parent,
+                  let parentQueryData = brokerURLsToQueryData[parentURL] else {
+                continue
+            }
+            let childRecordsCount = weeklyOptOuts(for: childQueryData).count
+            let parentRecordsCount = weeklyOptOuts(for: parentQueryData).count
+            let recordsCountDifference = childRecordsCount - parentRecordsCount
+
+            // If both values are zero there's no point sending the pixel
+            if recordsCountDifference <= 0 && value == 0 {
+                continue
+            }
+            handler.fire(.weeklyChildBrokerOrphanedOptOuts(dataBrokerName: childBrokerName,
+                                                           childParentRecordDifference: recordsCountDifference,
+                                                           calculatedOrphanedRecords: value))
+        }
+    }
+
+    func childBrokerURLsToOrphanedProfilesWeeklyCount(for data: [BrokerProfileQueryData]) -> [String: Int] {
+
+        let brokerURLsToQueryData = Dictionary(grouping: data, by: { $0.dataBroker.url })
+        let childBrokerURLsToQueryData = brokerURLsToQueryData.filter { (_, value: Array<BrokerProfileQueryData>) in
+            guard let first = value.first,
+                  first.dataBroker.parent != nil else {
+                return false
+            }
+            return true
+        }
+
+        let childBrokerURLsToOrphanedProfilesCount = childBrokerURLsToQueryData.mapValues { value in
+            guard let parent = value.first?.dataBroker.parent,
+                let parentsQueryData = brokerURLsToQueryData[parent] else {
+                return 0
+            }
+
+            let optOuts = weeklyOptOuts(for: value)
+            let parentBrokerOptOuts = weeklyOptOuts(for: parentsQueryData)
+
+            return orphanedProfilesCount(with: optOuts, parentOptOuts: parentBrokerOptOuts)
+        }
+
+        return childBrokerURLsToOrphanedProfilesCount
+    }
+
+    func orphanedProfilesCount(with childOptOuts: [OptOutJobData], parentOptOuts: [OptOutJobData]) -> Int {
+        let matchingCount = childOptOuts.reduce(0) { (partialResult: Int, optOut: OptOutJobData) in
+            let hasFoundParentMatch = parentOptOuts.contains { parentOptOut in
+                optOut.extractedProfile.doesMatchExtractedProfile(parentOptOut.extractedProfile)
+            }
+            return partialResult + (hasFoundParentMatch ? 1 : 0)
+        }
+        return childOptOuts.count - matchingCount
+    }
+}
+
 private extension Int {
     var toString: String {
         if self < 25 {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift

@@ -81,6 +81,8 @@ public enum DataBrokerProtectionPixels {
         static let numberOfNewRecordsFound = "num_new_found"
         static let numberOfReappereances = "num_reappeared"
         static let optOutSubmitSuccessRate = "optout_submit_success_rate"
+        static let childParentRecordDifference = "child-parent-record-difference"
+        static let calculatedOrphanedRecords = "calculated-orphaned-records"
     }
 
     case error(error: DataBrokerProtectionError, dataBroker: String)
@@ -211,6 +213,7 @@ public enum DataBrokerProtectionPixels {
     // Custom stats
     case customDataBrokerStatsOptoutSubmit(dataBrokerName: String, optOutSubmitSuccessRate: Double)
     case customGlobalStatsOptoutSubmit(optOutSubmitSuccessRate: Double)
+    case weeklyChildBrokerOrphanedOptOuts(dataBrokerName: String, childParentRecordDifference: Int, calculatedOrphanedRecords: Int)
 }
 
 extension DataBrokerProtectionPixels: PixelKitEvent {
@@ -345,8 +348,10 @@ extension DataBrokerProtectionPixels: PixelKitEvent {
         case .pixelTest: return "m_mac_dbp_configuration_pixel_test"
         case .failedToParsePrivacyConfig: return "m_mac_dbp_configuration_failed_to_parse"
 
+            // Various monitoring pixels
         case .customDataBrokerStatsOptoutSubmit: return "m_mac_dbp_databroker_custom_stats_optoutsubmit"
         case .customGlobalStatsOptoutSubmit: return "m_mac_dbp_custom_stats_optoutsubmit"
+        case .weeklyChildBrokerOrphanedOptOuts: return "m_mac_dbp_weekly_child-broker_orphaned-optouts"
         }
     }
 
@@ -522,6 +527,10 @@ extension DataBrokerProtectionPixels: PixelKitEvent {
                     Consts.optOutSubmitSuccessRate: String(optOutSubmitSuccessRate)]
         case .customGlobalStatsOptoutSubmit(let optOutSubmitSuccessRate):
             return [Consts.optOutSubmitSuccessRate: String(optOutSubmitSuccessRate)]
+        case .weeklyChildBrokerOrphanedOptOuts(let dataBrokerName, let childParentRecordDifference, let calculatedOrphanedRecords):
+            return [Consts.dataBrokerParamKey: dataBrokerName,
+                    Consts.childParentRecordDifference: String(childParentRecordDifference),
+                    Consts.calculatedOrphanedRecords: String(calculatedOrphanedRecords)]
         }
     }
 }
@@ -619,7 +628,8 @@ public class DataBrokerProtectionPixelsHandler: EventMapping<DataBrokerProtectio
                     .invalidPayload,
                     .pixelTest,
                     .customDataBrokerStatsOptoutSubmit,
-                    .customGlobalStatsOptoutSubmit:
+                    .customGlobalStatsOptoutSubmit,
+                    .weeklyChildBrokerOrphanedOptOuts:
 
                 PixelKit.fire(event)