Normalize domain names for Content Blocking (#91)

Task/Issue URL: https://app.asana.com/0/856498667320406/1202191081651325
iOS PR: duckduckgo/iOS#1134
macOS PR: N/A, test against develop.
What kind of version bump will this require?: Patch

Description:

Normalize domain names:

Make sure these are lower-case.
Punycode if needed.

Package.resolved

@@ -10,6 +10,15 @@
           "version": "1.1.0"
         }
       },
+      {
+        "package": "Punycode",
+        "repositoryURL": "https://github.com/gumob/PunycodeSwift.git",
+        "state": {
+          "branch": null,
+          "revision": "4356ec54e073741449640d3d50a1fd24fd1e1b8b",
+          "version": "2.1.0"
+        }
+      },
       {
         "package": "swift-argument-parser",
         "repositoryURL": "https://github.com/apple/swift-argument-parser",

Package.swift

@@ -15,7 +15,8 @@ let package = Package(
     ],
     dependencies: [
         .package(name: "GRDB", url: "https://github.com/duckduckgo/GRDB.swift.git", .exact("1.1.0")),
-        .package(url: "https://github.com/duckduckgo/TrackerRadarKit", .exact("1.0.3"))
+        .package(url: "https://github.com/duckduckgo/TrackerRadarKit", .exact("1.0.3")),
+        .package(name: "Punycode", url: "https://github.com/gumob/PunycodeSwift.git", .exact("2.1.0"))
     ],
     targets: [
 
@@ -24,6 +25,7 @@ let package = Package(
             dependencies: [
                 "GRDB",
                 "TrackerRadarKit",
+                .product(name: "Punnycode", package: "Punycode"),
                 "BloomFilterWrapper"
             ],
             exclude: [

Sources/BrowserServicesKit/Common/Extensions/StringExtension.swift

@@ -17,6 +17,7 @@
 //
 
 import Foundation
+import Punycode
 
 extension String {
 
@@ -57,3 +58,12 @@ extension String {
 
 }
 
+// MARK: - Punycode
+extension String {
+    public var punycodeEncodedHostname: String {
+        return self.split(separator: ".")
+            .map { String($0) }
+            .map { $0.idnaEncoded ?? $0 }
+            .joined(separator: ".")
+    }
+}

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift

@@ -116,7 +116,7 @@ public class DefaultContentBlockerRulesExceptionsSource: ContentBlockerRulesExce
             if entry.domains.contains("<all>") {
                 return TrackerException(rule: entry.rule, matching: .all)
             } else {
-                return TrackerException(rule: entry.rule, matching: .domains(entry.domains))
+                return TrackerException(rule: entry.rule, matching: .domains(entry.domains.normalizedDomainsForContentBlocking()))
             }
         }
     }

Sources/BrowserServicesKit/PrivacyConfig/AppPrivacyConfiguration.swift

@@ -27,19 +27,19 @@ public struct AppPrivacyConfiguration: PrivacyConfiguration {
     private let locallyUnprotected: DomainsProtectionStore
 
     public init(data: PrivacyConfigurationData,
-         identifier: String,
-         localProtection: DomainsProtectionStore) {
+                identifier: String,
+                localProtection: DomainsProtectionStore) {
         self.data = data
         self.identifier = identifier
         self.locallyUnprotected = localProtection
     }
 
     public var userUnprotectedDomains: [String] {
-        return Array(locallyUnprotected.unprotectedDomains)
+        return Array(locallyUnprotected.unprotectedDomains).normalizedDomainsForContentBlocking()
     }
 
     public var tempUnprotectedDomains: [String] {
-        return data.unprotectedTemporary.map { $0.domain }
+        return data.unprotectedTemporary.map { $0.domain }.normalizedDomainsForContentBlocking()
     }
 
     public var trackerAllowlist: PrivacyConfigurationData.TrackerAllowlistData {
@@ -84,7 +84,7 @@ public struct AppPrivacyConfiguration: PrivacyConfiguration {
     public func exceptionsList(forFeature featureKey: PrivacyFeature) -> [String] {
         guard let feature = data.features[featureKey.rawValue] else { return [] }
 
-        return feature.exceptions.map { $0.domain }
+        return feature.exceptions.map { $0.domain }.normalizedDomainsForContentBlocking()
     }
 
     public func isFeature(_ feature: PrivacyFeature, enabledForDomain domain: String?) -> Bool {
@@ -111,7 +111,7 @@ public struct AppPrivacyConfiguration: PrivacyConfiguration {
     public func isUserUnprotected(domain: String?) -> Bool {
         guard let domain = domain else { return false }
 
-        return locallyUnprotected.unprotectedDomains.contains(domain)
+        return userUnprotectedDomains.contains(domain)
     }
 
     public func isTempUnprotected(domain: String?) -> Bool {
@@ -146,11 +146,23 @@ public struct AppPrivacyConfiguration: PrivacyConfiguration {
     }
 
     public func userEnabledProtection(forDomain domain: String) {
-        locallyUnprotected.enableProtection(forDomain: domain)
+        let domainToRemove = locallyUnprotected.unprotectedDomains.first { unprotectedDomain in
+            unprotectedDomain.punycodeEncodedHostname.lowercased() == domain
+        }
+        locallyUnprotected.enableProtection(forDomain: domainToRemove ?? domain)
     }
 
     public func userDisabledProtection(forDomain domain: String) {
-        locallyUnprotected.disableProtection(forDomain: domain)
+        locallyUnprotected.disableProtection(forDomain: domain.punycodeEncodedHostname.lowercased())
     }
 
 }
+
+extension Array where Element == String {
+
+    func normalizedDomainsForContentBlocking() -> [String] {
+        map { domain in
+            domain.punycodeEncodedHostname.lowercased()
+        }
+    }
+}

Tests/BrowserServicesKitTests/Common/Extensions/StringExtensionTests.swift

@@ -17,7 +17,6 @@
 //
 
 import Foundation
-
 import XCTest
 @testable import BrowserServicesKit
 
@@ -64,5 +63,11 @@ final class StringExtensionTests: XCTestCase {
 
         XCTAssertEqual(normalizedString, "dax")
     }
-
+
+    func testWhenEmojisArePresentInDomains_ThenTheseCanBePunycoded() {
+
+        XCTAssertEqual("example.com".punycodeEncodedHostname, "example.com")
+        XCTAssertEqual("Dax🤔.com".punycodeEncodedHostname, "xn--dax-v153b.com")
+        XCTAssertEqual("🤔.com".punycodeEncodedHostname, "xn--wp9h.com")
+    }
 }