diff --git a/.gitmodules b/.gitmodules index 6816cfe92..f3afffd9b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "Sources/BrowserServicesKit/Resources/duckduckgo-autofill"] path = Sources/BrowserServicesKit/Resources/duckduckgo-autofill url = https://github.com/duckduckgo/duckduckgo-autofill +[submodule "Sources/BrowserServicesKit/Resources/content-scope-scripts"] + path = Sources/BrowserServicesKit/Resources/content-scope-scripts + url = https://github.com/duckduckgo/content-scope-scripts diff --git a/Package.resolved b/Package.resolved index a9e24374c..4fb493475 100644 --- a/Package.resolved +++ b/Package.resolved @@ -9,6 +9,24 @@ "revision": "9efe5a515acff8b73f69a31a65fc2bce2a823219", "version": "1.1.0" } + }, + { + "package": "swift-argument-parser", + "repositoryURL": "https://github.com/apple/swift-argument-parser", + "state": { + "branch": null, + "revision": "6b2aa2748a7881eebb9f84fb10c01293e15b52ca", + "version": "0.5.0" + } + }, + { + "package": "TrackerRadarKit", + "repositoryURL": "https://github.com/duckduckgo/TrackerRadarKit", + "state": { + "branch": null, + "revision": "5f4caf35b8418700a48c64c7c61eb43308c8dacc", + "version": "1.0.3" + } } ] }, diff --git a/Package.swift b/Package.swift index b30c3a4ec..ad2b37817 100644 --- a/Package.swift +++ b/Package.swift @@ -11,18 +11,33 @@ let package = Package( .macOS("10.15") ], products: [ - .library(name: "BrowserServicesKit", targets: ["BrowserServicesKit"]), + .library(name: "BrowserServicesKit", targets: ["BrowserServicesKit"]) ], dependencies: [ - .package(name: "GRDB", url: "https://github.com/duckduckgo/GRDB.swift.git", .exact("1.1.0")) + .package(name: "GRDB", url: "https://github.com/duckduckgo/GRDB.swift.git", .exact("1.1.0")), + .package(url: "https://github.com/duckduckgo/TrackerRadarKit", .exact("1.0.3")) ], targets: [ .target( name: "BrowserServicesKit", dependencies: [ - "GRDB" + "GRDB", + "TrackerRadarKit" ], exclude: [ + "Resources/content-scope-scripts/README.md", + "Resources/content-scope-scripts/package-lock.json", + "Resources/content-scope-scripts/package.json", + "Resources/content-scope-scripts/LICENSE.md", + "Resources/content-scope-scripts/src/", + "Resources/content-scope-scripts/unit-test/", + "Resources/content-scope-scripts/integration-test/", + "Resources/content-scope-scripts/scripts/", + "Resources/content-scope-scripts/inject/", + "Resources/content-scope-scripts/lib/", + "Resources/content-scope-scripts/build/chrome/", + "Resources/content-scope-scripts/build/firefox/", + "Resources/content-scope-scripts/build/integration/", "Resources/duckduckgo-autofill/Gruntfile.js", "Resources/duckduckgo-autofill/package.json", "Resources/duckduckgo-autofill/package-lock.json", @@ -34,10 +49,19 @@ let package = Package( "Resources/duckduckgo-autofill/jest.setup.js", "Resources/duckduckgo-autofill/dist/autofill-host-styles_chrome.css", "Resources/duckduckgo-autofill/jest.config.js", - "Resources/duckduckgo-autofill/jest-test-environment.js" + "Resources/duckduckgo-autofill/jest-test-environment.js", + "Resources/duckduckgo-autofill/scripts/release.js", + "Resources/duckduckgo-autofill/jesthtmlreporter.config.json", + "Resources/duckduckgo-autofill/types.d.ts", + "Resources/duckduckgo-autofill/tsconfig.json", + "Resources/duckduckgo-autofill/docs/real-world-html-tests.md", + "Resources/duckduckgo-autofill/docs/matcher-configuration.md" ], resources: [ - .process("Resources/duckduckgo-autofill/dist/autofill.js") + .process("Resources/duckduckgo-autofill/dist/autofill.js"), + .process("ContentBlocking/UserScripts/contentblockerrules.js"), + .process("ContentBlocking/UserScripts/surrogates.js"), + .process("Resources/content-scope-scripts/build/apple/contentScope.js") ]), .testTarget( name: "BrowserServicesKitTests", @@ -45,7 +69,8 @@ let package = Package( "BrowserServicesKit" ], resources: [ - .copy("UserScript/testUserScript.js") + .process("UserScript/testUserScript.js"), + .process("Resources") ]) ] ) diff --git a/Sources/BrowserServicesKit/Autofill/AutofillUserScript+Email.swift b/Sources/BrowserServicesKit/Autofill/AutofillUserScript+Email.swift index ebdcaa341..74b718193 100644 --- a/Sources/BrowserServicesKit/Autofill/AutofillUserScript+Email.swift +++ b/Sources/BrowserServicesKit/Autofill/AutofillUserScript+Email.swift @@ -34,7 +34,6 @@ public protocol AutofillEmailDelegate: AnyObject { extension AutofillUserScript { - func emailCheckSignedInStatus(_ message: WKScriptMessage, _ replyHandler: MessageReplyHandler) { let signedIn = emailDelegate?.autofillUserScriptDidRequestSignedInStatus(self) ?? false let signedInString = String(signedIn) @@ -97,5 +96,4 @@ extension AutofillUserScript { } } - } diff --git a/Sources/BrowserServicesKit/Autofill/AutofillUserScript+SecureVault.swift b/Sources/BrowserServicesKit/Autofill/AutofillUserScript+SecureVault.swift index 020b0425f..3ef8772ca 100644 --- a/Sources/BrowserServicesKit/Autofill/AutofillUserScript+SecureVault.swift +++ b/Sources/BrowserServicesKit/Autofill/AutofillUserScript+SecureVault.swift @@ -21,28 +21,155 @@ import WebKit public protocol AutofillSecureVaultDelegate: AnyObject { + func autofillUserScript(_: AutofillUserScript, didRequestAutoFillInitDataForDomain domain: String, completionHandler: @escaping ( + [SecureVaultModels.WebsiteAccount], + [SecureVaultModels.Identity], + [SecureVaultModels.CreditCard] + ) -> Void) + func autofillUserScript(_: AutofillUserScript, didRequestPasswordManagerForDomain domain: String) func autofillUserScript(_: AutofillUserScript, didRequestStoreCredentialsForDomain domain: String, username: String, password: String) func autofillUserScript(_: AutofillUserScript, didRequestAccountsForDomain domain: String, completionHandler: @escaping ([SecureVaultModels.WebsiteAccount]) -> Void) func autofillUserScript(_: AutofillUserScript, didRequestCredentialsForAccount accountId: Int64, completionHandler: @escaping (SecureVaultModels.WebsiteCredentials?) -> Void) + func autofillUserScript(_: AutofillUserScript, didRequestCreditCardWithId creditCardId: Int64, + completionHandler: @escaping (SecureVaultModels.CreditCard?) -> Void) + func autofillUserScript(_: AutofillUserScript, didRequestIdentityWithId identityId: Int64, + completionHandler: @escaping (SecureVaultModels.Identity?) -> Void) } extension AutofillUserScript { - struct RequestVaultAccountsResponse: Codable { + // MARK: - Response Objects - struct Account: Codable { - let id: Int64 - let username: String - let lastUpdated: TimeInterval + struct IdentityObject: Codable { + let id: Int64 + let title: String + + let firstName: String? + let middleName: String? + let lastName: String? + + let birthdayDay: Int? + let birthdayMonth: Int? + let birthdayYear: Int? + + let addressStreet: String? + let addressStreet2: String? + let addressCity: String? + let addressProvince: String? + let addressPostalCode: String? + let addressCountryCode: String? + + let phone: String? + let emailAddress: String? + + static func from(identity: SecureVaultModels.Identity) -> IdentityObject? { + guard let id = identity.id else { return nil } + + return IdentityObject(id: id, + title: identity.title, + firstName: identity.firstName, + middleName: identity.middleName, + lastName: identity.lastName, + birthdayDay: identity.birthdayDay, + birthdayMonth: identity.birthdayMonth, + birthdayYear: identity.birthdayYear, + addressStreet: identity.addressStreet, + addressStreet2: identity.addressStreet2, + addressCity: identity.addressCity, + addressProvince: identity.addressProvince, + addressPostalCode: identity.addressPostalCode, + addressCountryCode: identity.addressCountryCode, + phone: identity.homePhone, // Replace with single "phone number" column + emailAddress: identity.emailAddress) + } + } + + struct CreditCardObject: Codable { + let id: Int64 + let title: String + let displayNumber: String + + let cardName: String? + let cardNumber: String? + let cardSecurityCode: String? + let expirationMonth: Int? + let expirationYear: Int? + + static func from(card: SecureVaultModels.CreditCard) -> CreditCardObject? { + guard let id = card.id else { return nil } + + return CreditCardObject(id: id, + title: card.title, + displayNumber: card.displayName, + cardName: card.cardholderName, + cardNumber: card.cardNumber, + cardSecurityCode: card.cardSecurityCode, + expirationMonth: card.expirationMonth, + expirationYear: card.expirationYear) + } + + /// Provides a minimal summary of the card, suitable for presentation in the credit card selection list. This intentionally omits secure data, such as card number and cardholder name. + static func autofillInitializationValueFrom(card: SecureVaultModels.CreditCard) -> CreditCardObject? { + guard let id = card.id else { return nil } + + return CreditCardObject(id: id, + title: card.title, + displayNumber: card.displayName, + cardName: nil, + cardNumber: nil, + cardSecurityCode: nil, + expirationMonth: nil, + expirationYear: nil) + } + } + + struct CredentialObject: Codable { + let id: Int64 + let username: String + } + + // MARK: - Responses + + // swiftlint:disable nesting + struct RequestAutoFillInitDataResponse: Codable { + + struct AutofillInitSuccess: Codable { + let credentials: [CredentialObject] + let creditCards: [CreditCardObject] + let identities: [IdentityObject] } - let success: [Account] + let success: AutofillInitSuccess + let error: String? + + } + // swiftlint:enable nesting + + struct RequestAutoFillCreditCardResponse: Codable { + + let success: CreditCardObject + let error: String? + } + struct RequestAutoFillIdentityResponse: Codable { + + let success: IdentityObject + let error: String? + + } + + struct RequestVaultAccountsResponse: Codable { + + let success: [CredentialObject] + + } + + // swiftlint:disable nesting struct RequestVaultCredentialsResponse: Codable { struct Credential: Codable { @@ -54,6 +181,33 @@ extension AutofillUserScript { let success: Credential + } + // swiftlint:enable nesting + + // MARK: - Message Handlers + + func pmGetAutoFillInitData(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { + + let domain = hostProvider.hostForMessage(message) + vaultDelegate?.autofillUserScript(self, didRequestAutoFillInitDataForDomain: domain) { accounts, identities, cards in + let credentials: [CredentialObject] = accounts.compactMap { + guard let id = $0.id else { return nil } + return .init(id: id, username: $0.username) + } + + let identities: [IdentityObject] = identities.compactMap(IdentityObject.from(identity:)) + let cards: [CreditCardObject] = cards.compactMap(CreditCardObject.autofillInitializationValueFrom(card:)) + + let success = RequestAutoFillInitDataResponse.AutofillInitSuccess(credentials: credentials, + creditCards: cards, + identities: identities) + + let response = RequestAutoFillInitDataResponse(success: success, error: nil) + if let json = try? JSONEncoder().encode(response), let jsonString = String(data: json, encoding: .utf8) { + replyHandler(jsonString) + } + } + } func pmStoreCredentials(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { @@ -74,9 +228,9 @@ extension AutofillUserScript { func pmGetAccounts(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { vaultDelegate?.autofillUserScript(self, didRequestAccountsForDomain: hostProvider.hostForMessage(message)) { credentials in - let credentials: [RequestVaultAccountsResponse.Account] = credentials.compactMap { + let credentials: [CredentialObject] = credentials.compactMap { guard let id = $0.id else { return nil } - return .init(id: id, username: $0.username, lastUpdated: $0.lastUpdated.timeIntervalSince1970) + return .init(id: id, username: $0.username) } let response = RequestVaultAccountsResponse(success: credentials) @@ -110,6 +264,54 @@ extension AutofillUserScript { } } + func pmGetCreditCard(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { + guard let body = message.body as? [String: Any], + let id = body["id"] as? String, + let cardId = Int64(id) else { + return + } + + vaultDelegate?.autofillUserScript(self, didRequestCreditCardWithId: Int64(cardId)) { + guard let card = $0, let cardObject = CreditCardObject.from(card: card) else { return } + + let response = RequestAutoFillCreditCardResponse(success: cardObject, error: nil) + + if let json = try? JSONEncoder().encode(response), let jsonString = String(data: json, encoding: .utf8) { + replyHandler(jsonString) + } + } + } + + func pmGetIdentity(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { + guard let body = message.body as? [String: Any], + let id = body["id"] as? String, + let accountId = Int64(id) else { + return + } + + vaultDelegate?.autofillUserScript(self, didRequestIdentityWithId: Int64(accountId)) { + guard let identity = $0, let identityObject = IdentityObject.from(identity: identity) else { return } + + let response = RequestAutoFillIdentityResponse(success: identityObject, error: nil) + + if let json = try? JSONEncoder().encode(response), let jsonString = String(data: json, encoding: .utf8) { + replyHandler(jsonString) + } + } + } + + // MARK: Open Management Views + + func pmOpenManageCreditCards(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { + vaultDelegate?.autofillUserScript(self, didRequestPasswordManagerForDomain: hostProvider.hostForMessage(message)) + replyHandler(nil) + } + + func pmOpenManageIdentities(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { + vaultDelegate?.autofillUserScript(self, didRequestPasswordManagerForDomain: hostProvider.hostForMessage(message)) + replyHandler(nil) + } + func pmOpenManagePasswords(_ message: WKScriptMessage, _ replyHandler: @escaping MessageReplyHandler) { vaultDelegate?.autofillUserScript(self, didRequestPasswordManagerForDomain: hostProvider.hostForMessage(message)) replyHandler(nil) diff --git a/Sources/BrowserServicesKit/Autofill/AutofillUserScript.swift b/Sources/BrowserServicesKit/Autofill/AutofillUserScript.swift index 9cc965d9b..05c90dcae 100644 --- a/Sources/BrowserServicesKit/Autofill/AutofillUserScript.swift +++ b/Sources/BrowserServicesKit/Autofill/AutofillUserScript.swift @@ -32,11 +32,17 @@ public class AutofillUserScript: NSObject, UserScript { case emailHandlerGetAddresses case emailHandlerCheckAppSignedInStatus + case pmHandlerGetAutofillInitData + case pmHandlerStoreCredentials case pmHandlerGetAccounts case pmHandlerGetAutofillCredentials - case pmHandlerOpenManagePasswords + case pmHandlerGetIdentity + case pmHandlerGetCreditCard + case pmHandlerOpenManageCreditCards + case pmHandlerOpenManageIdentities + case pmHandlerOpenManagePasswords } public weak var emailDelegate: AutofillEmailDelegate? @@ -76,9 +82,17 @@ public class AutofillUserScript: NSObject, UserScript { case .emailHandlerGetAddresses: return emailGetAddresses case .emailHandlerCheckAppSignedInStatus: return emailCheckSignedInStatus + case .pmHandlerGetAutofillInitData: return pmGetAutoFillInitData + case .pmHandlerStoreCredentials: return pmStoreCredentials case .pmHandlerGetAccounts: return pmGetAccounts case .pmHandlerGetAutofillCredentials: return pmGetAutofillCredentials + + case .pmHandlerGetIdentity: return pmGetIdentity + case .pmHandlerGetCreditCard: return pmGetCreditCard + + case .pmHandlerOpenManageCreditCards: return pmOpenManageCreditCards + case .pmHandlerOpenManageIdentities: return pmOpenManageIdentities case .pmHandlerOpenManagePasswords: return pmOpenManagePasswords } diff --git a/Sources/BrowserServicesKit/Common/Extensions/HashExtension.swift b/Sources/BrowserServicesKit/Common/Extensions/HashExtension.swift new file mode 100644 index 000000000..45639ce34 --- /dev/null +++ b/Sources/BrowserServicesKit/Common/Extensions/HashExtension.swift @@ -0,0 +1,50 @@ +// +// HashExtension.swift +// DuckDuckGo +// +// Copyright © 2019 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import CommonCrypto + +extension Data { + + private typealias Algorithm = (UnsafeRawPointer?, CC_LONG, UnsafeMutablePointer?) -> UnsafeMutablePointer? + + public var sha1: String { + return hash(algorithm: CC_SHA1, length: CC_SHA1_DIGEST_LENGTH) + } + + public var sha256: String { + return hash(algorithm: CC_SHA256, length: CC_SHA256_DIGEST_LENGTH) + } + + private func hash(algorithm: Algorithm, length: Int32) -> String { + var hash = [UInt8](repeating: 0, count: Int(length)) + let dataBytes = [UInt8](self) + _ = algorithm(dataBytes, CC_LONG(self.count), &hash) + return hash.map { String(format: "%02x", $0) }.joined() + } +} + +extension String { + + public var sha1: String { + let dataBytes = data(using: .utf8)! + return dataBytes.sha1 + } + +} diff --git a/Sources/BrowserServicesKit/Common/Extensions/StringExtension.swift b/Sources/BrowserServicesKit/Common/Extensions/StringExtension.swift index a7fbb7876..0407afedb 100644 --- a/Sources/BrowserServicesKit/Common/Extensions/StringExtension.swift +++ b/Sources/BrowserServicesKit/Common/Extensions/StringExtension.swift @@ -20,6 +20,10 @@ import Foundation extension String { + public func trimWhitespace() -> String { + return trimmingCharacters(in: .whitespacesAndNewlines) + } + func dropping(prefix: String) -> String { return hasPrefix(prefix) ? String(dropFirst(prefix.count)) : self } diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerDebugEvents.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerDebugEvents.swift new file mode 100644 index 000000000..66d88258d --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerDebugEvents.swift @@ -0,0 +1,41 @@ +// +// ContentBlockerDebugEvents.swift +// DuckDuckGo +// +// Copyright © 2019 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +public enum ContentBlockerDebugEvents { + + struct Parameters { + static let etag = "etag" + static let errorDescription = "error_desc" + } + + case trackerDataParseFailed + case trackerDataReloadFailed + case trackerDataCouldNotBeLoaded + case privacyConfigurationReloadFailed + case privacyConfigurationParseFailed + case privacyConfigurationCouldNotBeLoaded + + case contentBlockingTDSCompilationFailed + case contentBlockingTempListCompilationFailed + case contentBlockingAllowListCompilationFailed + case contentBlockingUnpSitesCompilationFailed + case contentBlockingFallbackCompilationFailed +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesIdentifier.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesIdentifier.swift new file mode 100644 index 000000000..bc0a84133 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesIdentifier.swift @@ -0,0 +1,105 @@ +// +// ContentBlockerRulesIdentifier.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +public class ContentBlockerRulesIdentifier: Equatable { + + private let name: String + private let tdsEtag: String + private let tempListEtag: String + private let allowListEtag: String + private let unprotectedSitesHash: String + + public var stringValue: String { + return name + tdsEtag + tempListEtag + unprotectedSitesHash + } + + public struct Difference: OptionSet { + public let rawValue: Int + + public init(rawValue: Int) { + self.rawValue = rawValue + } + + public static let tdsEtag = Difference(rawValue: 1 << 0) + public static let tempListEtag = Difference(rawValue: 1 << 1) + public static let allowListEtag = Difference(rawValue: 1 << 2) + public static let unprotectedSites = Difference(rawValue: 1 << 3) + + public static let all: Difference = [.tdsEtag, .tempListEtag, .allowListEtag, .unprotectedSites] + } + + private class func normalize(identifier: String?) -> String { + // Ensure identifier is in double quotes + guard var identifier = identifier else { + return "\"\"" + } + + if !identifier.hasSuffix("\"") { + identifier += "\"" + } + + if !identifier.hasPrefix("\"") || identifier.count == 1 { + identifier = "\"" + identifier + } + + return identifier + } + + public class func hash(domains: [String]?) -> String { + guard let domains = domains, !domains.isEmpty else { + return "" + } + + return domains.joined().sha1 + } + + public init(name: String, tdsEtag: String, tempListEtag: String?, allowListEtag: String?, unprotectedSitesHash: String?) { + + self.name = Self.normalize(identifier: name) + self.tdsEtag = Self.normalize(identifier: tdsEtag) + self.tempListEtag = Self.normalize(identifier: tempListEtag) + self.allowListEtag = Self.normalize(identifier: allowListEtag) + self.unprotectedSitesHash = Self.normalize(identifier: unprotectedSitesHash) + } + + public func compare(with id: ContentBlockerRulesIdentifier) -> Difference { + + var result = Difference() + if tdsEtag != id.tdsEtag { + result.insert(.tdsEtag) + } + if tempListEtag != id.tempListEtag { + result.insert(.tempListEtag) + } + if allowListEtag != id.allowListEtag { + result.insert(.allowListEtag) + } + if unprotectedSitesHash != id.unprotectedSitesHash { + result.insert(.unprotectedSites) + } + + return result + } + + public static func == (lhs: ContentBlockerRulesIdentifier, rhs: ContentBlockerRulesIdentifier) -> Bool { + return lhs.compare(with: rhs).isEmpty + } +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift new file mode 100644 index 000000000..31c5a4ae4 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift @@ -0,0 +1,382 @@ +// +// ContentBlockerRulesManager.swift +// DuckDuckGo +// +// Copyright © 2020 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import WebKit +import os.log +import TrackerRadarKit + +public protocol ContentBlockerRulesUpdating { + + func rulesManager(_ manager: ContentBlockerRulesManager, + didUpdateRules: [ContentBlockerRulesManager.Rules], + changes: [String: ContentBlockerRulesIdentifier.Difference], + completionTokens: [ContentBlockerRulesManager.CompletionToken]) +} + +/** + Encapsulates compilation steps for a single Task + */ +private class CompilationTask { + typealias Completion = (_ success: Bool) -> Void + let workQueue: DispatchQueue + let rulesList: ContentBlockerRulesList + let sourceManager: ContentBlockerRulesSourceManager + let logger: OSLog + + var completed: Bool { result != nil || compilationImpossible } + var compilationImpossible = false + var result: (compiledRulesList: WKContentRuleList, model: ContentBlockerRulesSourceModel)? + + init(workQueue: DispatchQueue, + rulesList: ContentBlockerRulesList, + sourceManager: ContentBlockerRulesSourceManager, + logger: OSLog = .disabled) { + self.workQueue = workQueue + self.rulesList = rulesList + self.sourceManager = sourceManager + self.logger = logger + } + + func start(completionHandler: @escaping Completion) { + + guard let model = sourceManager.makeModel() else { + compilationImpossible = true + completionHandler(false) + return + } + + // Delegate querying to main thread - crashes were observed in background. + DispatchQueue.main.async { + WKContentRuleListStore.default()?.lookUpContentRuleList(forIdentifier: model.rulesIdentifier.stringValue, + completionHandler: { ruleList, _ in + if let ruleList = ruleList { + self.compilationSucceded(with: ruleList, model: model, completionHandler: completionHandler) + } else { + self.workQueue.async { + self.compile(model: model, completionHandler: completionHandler) + } + } + }) + } + } + + private func compilationSucceded(with compiledRulesList: WKContentRuleList, + model: ContentBlockerRulesSourceModel, + completionHandler: @escaping Completion) { + workQueue.async { + self.result = (compiledRulesList, model) + completionHandler(true) + } + } + + private func compilationFailed(for model: ContentBlockerRulesSourceModel, + with error: Error, + completionHandler: @escaping Completion) { + workQueue.async { + os_log("Failed to compile %{public}s rules %{public}s", log: self.logger, type: .error, self.rulesList.name, error.localizedDescription) + + // Retry after marking failed state in the source + self.sourceManager.compilationFailed(for: model, with: error) + + if let newModel = self.sourceManager.makeModel() { + self.compile(model: newModel, completionHandler: completionHandler) + } else { + self.compilationImpossible = true + completionHandler(false) + } + } + } + + private func compile(model: ContentBlockerRulesSourceModel, + completionHandler: @escaping Completion) { + os_log("Starting CBR compilation for %{public}s", log: logger, type: .default, rulesList.name) + + let builder = ContentBlockerRulesBuilder(trackerData: model.tds) + let rules = builder.buildRules(withExceptions: model.unprotectedSites, + andTemporaryUnprotectedDomains: model.tempList, + andTrackerAllowlist: model.allowList) + + let data: Data + do { + data = try JSONEncoder().encode(rules) + } catch { + os_log("Failed to encode content blocking rules %{public}s", log: logger, type: .error, rulesList.name) + compilationFailed(for: model, with: error, completionHandler: completionHandler) + return + } + + let ruleList = String(data: data, encoding: .utf8)! + WKContentRuleListStore.default().compileContentRuleList(forIdentifier: model.rulesIdentifier.stringValue, + encodedContentRuleList: ruleList) { ruleList, error in + + if let ruleList = ruleList { + self.compilationSucceded(with: ruleList, model: model, completionHandler: completionHandler) + } else if let error = error { + self.compilationFailed(for: model, with: error, completionHandler: completionHandler) + } else { + assertionFailure("Rule list has not been returned properly by the engine") + } + } + } +} + +/** + Manages creation of Content Blocker rules from `ContentBlockerRulesSource`. + */ +public class ContentBlockerRulesManager { + + public typealias CompletionToken = String + + enum State { + case idle // Waiting for work + case recompiling(currentTokens: [CompletionToken]) // Executing work + case recompilingAndScheduled(currentTokens: [CompletionToken], pendingTokens: [CompletionToken]) // New work has been requested while one is currently being executed + } + + /** + Encapsulates information about the result of the task compilation. + */ + public struct Rules { + public let name: String + public let rulesList: WKContentRuleList + public let trackerData: TrackerData + public let encodedTrackerData: String + public let etag: String + public let identifier: ContentBlockerRulesIdentifier + } + + private let rulesSource: ContentBlockerRulesListsSource + private let exceptionsSource: ContentBlockerRulesExceptionsSource + private let updateListener: ContentBlockerRulesUpdating? + private let errorReporting: EventMapping? + private let logger: OSLog + + // Public only for tests + public var sourceManagers = [String: ContentBlockerRulesSourceManager]() + + private var currentTasks = [CompilationTask]() + + private let workQueue = DispatchQueue(label: "ContentBlockerManagerQueue", qos: .userInitiated) + + public init(rulesSource: ContentBlockerRulesListsSource, + exceptionsSource: ContentBlockerRulesExceptionsSource, + updateListener: ContentBlockerRulesUpdating, + errorReporting: EventMapping? = nil, + logger: OSLog = .disabled) { + self.rulesSource = rulesSource + self.exceptionsSource = exceptionsSource + self.updateListener = updateListener + self.errorReporting = errorReporting + self.logger = logger + + requestCompilation(token: "") + } + + /** + Variables protected by this lock: + - state + - currentRules + */ + private let lock = NSLock() + + private var state = State.idle + + private var _currentRules = [Rules]() + public private(set) var currentRules: [Rules] { + get { + lock.lock(); defer { lock.unlock() } + return _currentRules + } + set { + lock.lock() + self._currentRules = newValue + lock.unlock() + } + } + + @discardableResult + public func scheduleCompilation() -> CompletionToken { + let token = UUID().uuidString + workQueue.async { + self.requestCompilation(token: token) + } + return token + } + + private func requestCompilation(token: CompletionToken) { + os_log("Requesting compilation...", log: logger, type: .default) + lock.lock() + guard case .idle = state else { + if case .recompiling(let tokens) = state { + // Schedule reload + state = .recompilingAndScheduled(currentTokens: tokens, pendingTokens: [token]) + } else if case .recompilingAndScheduled(let currentTokens, let pendingTokens) = state { + state = .recompilingAndScheduled(currentTokens: currentTokens, pendingTokens: pendingTokens + [token]) + } + lock.unlock() + return + } + + state = .recompiling(currentTokens: [token]) + lock.unlock() + + startCompilationProcess() + } + + private func startCompilationProcess() { + // Prepare compilation tasks based on the sources + currentTasks = rulesSource.contentBlockerRulesLists.map({ rulesList in + + let sourceManager: ContentBlockerRulesSourceManager + if let manager = self.sourceManagers[rulesList.name] { + // Update rules list + manager.rulesList = rulesList + sourceManager = manager + } else { + sourceManager = ContentBlockerRulesSourceManager(rulesList: rulesList, + exceptionsSource: self.exceptionsSource, + errorReporting: self.errorReporting) + self.sourceManagers[rulesList.name] = sourceManager + } + return CompilationTask(workQueue: workQueue, rulesList: rulesList, sourceManager: sourceManager) + }) + + executeNextTask() + } + + private func executeNextTask() { + if let nextTask = currentTasks.first(where: { !$0.completed }) { + nextTask.start { _ in + self.executeNextTask() + } + } else { + compilationCompleted() + } + } + + static func extractSurrogates(from tds: TrackerData) -> TrackerData { + + let trackers = tds.trackers.filter { pair in + return pair.value.rules?.first(where: { rule in + rule.surrogate != nil + }) != nil + } + + var domains = [TrackerData.TrackerDomain: TrackerData.EntityName]() + var entities = [TrackerData.EntityName: Entity]() + for tracker in trackers { + if let entityName = tds.domains[tracker.key] { + domains[tracker.key] = entityName + entities[entityName] = tds.entities[entityName] + } + } + + var cnames = [TrackerData.CnameDomain: TrackerData.TrackerDomain]() + if let tdsCnames = tds.cnames { + for pair in tdsCnames { + for domain in domains.keys { + if pair.value.hasSuffix(domain) { + cnames[pair.key] = pair.value + break + } + } + } + } + + return TrackerData(trackers: trackers, entities: entities, domains: domains, cnames: cnames) + } + + private func compilationCompleted() { + + var changes = [String: ContentBlockerRulesIdentifier.Difference]() + + lock.lock() + + let newRules: [Rules] = currentTasks.compactMap { task in + guard let result = task.result else { + os_log("Failed to complete task %{public}s ", log: self.logger, type: .error, task.rulesList.name) + return nil + } + + let surrogateTDS = Self.extractSurrogates(from: result.model.tds) + let encodedData = try? JSONEncoder().encode(surrogateTDS) + let encodedTrackerData = String(data: encodedData!, encoding: .utf8)! + + let diff: ContentBlockerRulesIdentifier.Difference + if let id = _currentRules.first(where: {$0.name == task.rulesList.name })?.identifier { + diff = id.compare(with: result.model.rulesIdentifier) + } else { + diff = result.model.rulesIdentifier.compare(with: ContentBlockerRulesIdentifier(name: task.rulesList.name, + tdsEtag: "", + tempListEtag: nil, + allowListEtag: nil, + unprotectedSitesHash: nil)) + } + + changes[task.rulesList.name] = diff + + return Rules(name: task.rulesList.name, + rulesList: result.compiledRulesList, + trackerData: result.model.tds, + encodedTrackerData: encodedTrackerData, + etag: result.model.tdsIdentifier, + identifier: result.model.rulesIdentifier) + } + + _currentRules = newRules + + let currentIdentifiers: [String] = newRules.map { $0.identifier.stringValue } + + var completionTokens = [CompletionToken]() + if case .recompilingAndScheduled(let currentTokens, let pendingTokens) = state { + // New work has been scheduled - prepare for execution. + workQueue.async { + self.startCompilationProcess() + } + + completionTokens = currentTokens + state = .recompiling(currentTokens: pendingTokens) + } else if case .recompiling(let currentTokens) = state { + completionTokens = currentTokens + state = .idle + } + + lock.unlock() + + DispatchQueue.main.async { + self.updateListener?.rulesManager(self, + didUpdateRules: newRules, + changes: changes, + completionTokens: completionTokens) + + WKContentRuleListStore.default()?.getAvailableContentRuleListIdentifiers({ ids in + guard let ids = ids else { return } + + var idsSet = Set(ids) + idsSet.subtract(currentIdentifiers) + + for id in idsSet { + WKContentRuleListStore.default()?.removeContentRuleList(forIdentifier: id) { _ in } + } + }) + } + } + +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift new file mode 100644 index 000000000..07d525e53 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift @@ -0,0 +1,124 @@ +// +// ContentBlockerRulesSource.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import TrackerRadarKit + +/** + Represents all sources used to build Content Blocking Rules. + */ +public protocol ContentBlockerRulesListsSource { + + var contentBlockerRulesLists: [ContentBlockerRulesList] { get } +} + +/** + Represents sources used to prepare exceptions to content blocking Rules. + */ +public protocol ContentBlockerRulesExceptionsSource { + + var tempListEtag: String { get } + var tempList: [String] { get } + var allowListEtag: String { get } + var allowList: [TrackerException] { get } + var unprotectedSites: [String] { get } +} + +public struct ContentBlockerRulesList { + + public let trackerData: TrackerDataManager.DataSet? + public let fallbackTrackerData: TrackerDataManager.DataSet + + public let name: String + + public init(name: String, + trackerData: TrackerDataManager.DataSet?, + fallbackTrackerData: TrackerDataManager.DataSet) { + self.name = name + self.trackerData = trackerData + self.fallbackTrackerData = fallbackTrackerData + } +} + +open class DefaultContentBlockerRulesListsSource: ContentBlockerRulesListsSource { + + public struct Constants { + public static let trackerDataSetRulesListName = "TrackerDataSet" + } + + private let trackerDataManger: TrackerDataManager + + public init(trackerDataManger: TrackerDataManager) { + self.trackerDataManger = trackerDataManger + } + + open var contentBlockerRulesLists: [ContentBlockerRulesList] { + return [ContentBlockerRulesList(name: Constants.trackerDataSetRulesListName, + trackerData: trackerDataManger.fetchedData, + fallbackTrackerData: trackerDataManger.embeddedData)] + } +} + +public class DefaultContentBlockerRulesExceptionsSource: ContentBlockerRulesExceptionsSource { + + let privacyConfigManager: PrivacyConfigurationManager + + public init(privacyConfigManager: PrivacyConfigurationManager) { + self.privacyConfigManager = privacyConfigManager + } + + public var tempListEtag: String { + return privacyConfigManager.privacyConfig.identifier + } + + public var tempList: [String] { + let config = privacyConfigManager.privacyConfig + var tempUnprotected = config.tempUnprotectedDomains.filter { !$0.trimWhitespace().isEmpty } + tempUnprotected.append(contentsOf: config.exceptionsList(forFeature: .contentBlocking)) + return tempUnprotected + } + + public var allowListEtag: String { + return privacyConfigManager.privacyConfig.identifier + } + + public var allowList: [TrackerException] { + return Self.transform(allowList: privacyConfigManager.privacyConfig.trackerAllowlist) + } + + public var unprotectedSites: [String] { + return privacyConfigManager.privacyConfig.userUnprotectedDomains + } + + public class func transform(allowList: PrivacyConfigurationData.TrackerAllowlistData) -> [TrackerException] { + + let trackerRules = allowList.values.reduce(into: []) { partialResult, next in + partialResult.append(contentsOf: next) + } + + return trackerRules.map { entry in + if entry.domains.contains("") { + return TrackerException(rule: entry.rule, matching: .all) + } else { + return TrackerException(rule: entry.rule, matching: .domains(entry.domains)) + } + } + } + +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSourceManager.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSourceManager.swift new file mode 100644 index 000000000..103964ae3 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSourceManager.swift @@ -0,0 +1,211 @@ +// +// ContentBlockerRulesSourceManager.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import TrackerRadarKit + +/** + Encapsulates revision of the Content Blocker Rules source - id/etag of each of the resources used for compilation. + */ +public class ContentBlockerRulesSourceIdentifiers { + + public let name: String + public let tdsIdentifier: String + + public internal(set) var tempListIdentifier: String? + + public internal(set) var allowListIdentifier: String? + + public internal(set) var unprotectedSitesIdentifier: String? + + init(name: String, tdsIdentfier: String) { + self.name = name + self.tdsIdentifier = tdsIdentfier + } + + public var rulesIdentifier: ContentBlockerRulesIdentifier { + ContentBlockerRulesIdentifier(name: name, + tdsEtag: tdsIdentifier, + tempListEtag: tempListIdentifier, + allowListEtag: allowListIdentifier, + unprotectedSitesHash: unprotectedSitesIdentifier) + } +} + +/** + Model used to compile Content Blocking Rules along with Identifiers. + */ +public class ContentBlockerRulesSourceModel: ContentBlockerRulesSourceIdentifiers { + + let tds: TrackerData + + var tempList = [String]() + + var allowList = [TrackerException]() + + var unprotectedSites = [String]() + + init(name: String, tdsIdentfier: String, tds: TrackerData) { + self.tds = tds + super.init(name: name, tdsIdentfier: tdsIdentfier) + } +} + +/** + Manages sources that are used to compile Content Blocking Rules, handles possible broken state by filtering out sources that are potentially corrupted. + */ +public class ContentBlockerRulesSourceManager { + + /** + Data source for all of the exception info used during compilation. + */ + private let exceptionsSource: ContentBlockerRulesExceptionsSource + + var rulesList: ContentBlockerRulesList + + /** + Identifiers of sources that have caused compilation process to fail. + */ + public private(set) var brokenSources: ContentBlockerRulesSourceIdentifiers? + public private(set) var fallbackTDSFailure = false + + private let errorReporting: EventMapping? + + init(rulesList: ContentBlockerRulesList, + exceptionsSource: ContentBlockerRulesExceptionsSource, + errorReporting: EventMapping? = nil) { + self.rulesList = rulesList + self.exceptionsSource = exceptionsSource + self.errorReporting = errorReporting + } + + /** + Create Source Model based on data source and known broken sources. + + This method takes into account changes to `dataSource` that could fix previously corrupted data set - in such case `brokenSources` state is updated. + */ + func makeModel() -> ContentBlockerRulesSourceModel? { + guard !fallbackTDSFailure else { + return nil + } + + // Fetch identifiers up-front + let tempListIdentifier = exceptionsSource.tempListEtag + let allowListIdentifier = exceptionsSource.allowListEtag + let unprotectedSites = exceptionsSource.unprotectedSites + let unprotectedSitesIdentifier = ContentBlockerRulesIdentifier.hash(domains: unprotectedSites) + + // In case of any broken input that has been changed, reset the broken state and retry full compilation + if (brokenSources?.tempListIdentifier != nil && brokenSources?.tempListIdentifier != tempListIdentifier) || + brokenSources?.unprotectedSitesIdentifier != nil && brokenSources?.unprotectedSitesIdentifier != unprotectedSitesIdentifier || + brokenSources?.allowListIdentifier != nil && brokenSources?.allowListIdentifier != allowListIdentifier { + brokenSources = nil + } + + // Check which Tracker Data Set to use - fallback to embedded one in case of any issues. + let result: ContentBlockerRulesSourceModel + if let trackerData = rulesList.trackerData, + trackerData.etag != brokenSources?.tdsIdentifier { + result = ContentBlockerRulesSourceModel(name: rulesList.name, + tdsIdentfier: trackerData.etag, + tds: trackerData.tds) + } else { + result = ContentBlockerRulesSourceModel(name: rulesList.name, + tdsIdentfier: rulesList.fallbackTrackerData.etag, + tds: rulesList.fallbackTrackerData.tds) + } + + if tempListIdentifier != brokenSources?.tempListIdentifier { + let tempListDomains = exceptionsSource.tempList + if !tempListDomains.isEmpty { + result.tempListIdentifier = tempListIdentifier + result.tempList = tempListDomains + } + } + + if allowListIdentifier != brokenSources?.allowListIdentifier { + let allowList = exceptionsSource.allowList + if !allowList.isEmpty { + result.allowListIdentifier = allowListIdentifier + result.allowList = allowList + } + } + + if unprotectedSitesIdentifier != brokenSources?.unprotectedSitesIdentifier { + if !unprotectedSites.isEmpty { + result.unprotectedSitesIdentifier = unprotectedSitesIdentifier + result.unprotectedSites = unprotectedSites + } + } + + return result + } + + /** + Process information about last failed compilation in order to update `brokenSources` state. + */ + func compilationFailed(for input: ContentBlockerRulesSourceIdentifiers, with error: Error) { + + if input.tdsIdentifier != rulesList.fallbackTrackerData.etag { + // We failed compilation for non-embedded TDS, marking it as broken. + brokenSources = ContentBlockerRulesSourceIdentifiers(name: rulesList.name, + tdsIdentfier: input.tdsIdentifier) + + errorReporting?.fire(.contentBlockingTDSCompilationFailed, + scope: input.name, + error: error, + parameters: [ContentBlockerDebugEvents.Parameters.etag: input.tdsIdentifier]) + } else if input.tempListIdentifier != nil { + brokenSources?.tempListIdentifier = input.tempListIdentifier + errorReporting?.fire(.contentBlockingTempListCompilationFailed, + scope: input.name, + error: error, + parameters: [ContentBlockerDebugEvents.Parameters.etag: input.tempListIdentifier ?? "empty"]) + } else if input.allowListIdentifier != nil { + brokenSources?.allowListIdentifier = input.allowListIdentifier + errorReporting?.fire(.contentBlockingAllowListCompilationFailed, + scope: input.name, + error: error, + parameters: [ContentBlockerDebugEvents.Parameters.etag: input.allowListIdentifier ?? "empty"]) + } else if input.unprotectedSitesIdentifier != nil { + brokenSources?.unprotectedSitesIdentifier = input.unprotectedSitesIdentifier + errorReporting?.fire(.contentBlockingUnpSitesCompilationFailed, + scope: input.name, + error: error) + } else { + // We failed for embedded data, this is unlikely. + // Include description - why built-in version of the TDS has failed to compile? + let error = error as NSError + let errorDesc = (error.userInfo[NSHelpAnchorErrorKey] as? String) ?? "missing" + let params = [ContentBlockerDebugEvents.Parameters.errorDescription: errorDesc.isEmpty ? "empty" : errorDesc] + + errorReporting?.fire(.contentBlockingFallbackCompilationFailed, + scope: input.name, + error: error, + parameters: params, + onComplete: { _ in + if input.name == DefaultContentBlockerRulesListsSource.Constants.trackerDataSetRulesListName { + fatalError("Could not compile embedded rules list") + } + }) + + fallbackTDSFailure = true + } + } +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/DetectedTracker.swift b/Sources/BrowserServicesKit/ContentBlocking/DetectedTracker.swift new file mode 100644 index 000000000..0186e9672 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/DetectedTracker.swift @@ -0,0 +1,62 @@ +// +// DetectedTracker.swift +// DuckDuckGo +// +// Copyright © 2017 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import TrackerRadarKit + +// Populated with relevant info at the point of detection. +public struct DetectedTracker: Encodable { + + public let url: String + public let knownTracker: KnownTracker? + public let entity: Entity? + public let blocked: Bool + public let pageUrl: String + + public init(url: String, knownTracker: KnownTracker?, entity: Entity?, blocked: Bool, pageUrl: String) { + self.url = url + self.knownTracker = knownTracker + self.entity = entity + self.blocked = blocked + self.pageUrl = pageUrl + } + + public var domain: String? { + return URL(string: url)?.host + } + + public var networkNameForDisplay: String { + return entity?.displayName ?? domain ?? url + } + +} + +extension DetectedTracker: Hashable, Equatable { + + public static func == (lhs: DetectedTracker, rhs: DetectedTracker) -> Bool { + return ((lhs.entity != nil || rhs.entity != nil) && lhs.entity?.displayName == rhs.entity?.displayName) + && lhs.domain ?? "" == rhs.domain ?? "" + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(self.entity?.displayName) + hasher.combine(self.domain) + } + +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/DomainsProtectionStore.swift b/Sources/BrowserServicesKit/ContentBlocking/DomainsProtectionStore.swift new file mode 100644 index 000000000..fd0bfc13f --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/DomainsProtectionStore.swift @@ -0,0 +1,29 @@ +// +// DomainsProtectionStore.swift +// DuckDuckGo +// +// Copyright © 2017 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +public protocol DomainsProtectionStore: AnyObject { + + var unprotectedDomains: Set { get } + + func disableProtection(forDomain domain: String) + + func enableProtection(forDomain domain: String) +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/EventMapping.swift b/Sources/BrowserServicesKit/ContentBlocking/EventMapping.swift new file mode 100644 index 000000000..5e1e5f32c --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/EventMapping.swift @@ -0,0 +1,37 @@ +// +// EventMapping.swift +// DuckDuckGo +// +// Copyright © 2019 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +public class EventMapping { + public typealias Mapping = (_ event: BSKEvent, + _ scope: String?, + _ error: Error?, + _ params: [String: String]?, _ onComplete: @escaping (Error?) -> Void) -> Void + + private let eventMapper: Mapping + + public init(mapping: @escaping Mapping) { + eventMapper = mapping + } + + public func fire(_ event: BSKEvent, scope: String? = nil, error: Error? = nil, parameters: [String: String]? = nil, onComplete: @escaping (Error?) -> Void = {_ in }) { + eventMapper(event, scope, error, parameters, onComplete) + } +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/TrackerDataManager.swift b/Sources/BrowserServicesKit/ContentBlocking/TrackerDataManager.swift new file mode 100644 index 000000000..4020ff520 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/TrackerDataManager.swift @@ -0,0 +1,126 @@ +// +// TrackerDataManager.swift +// DuckDuckGo +// +// Copyright © 2019 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import TrackerRadarKit + +public protocol TrackerDataProvider { + + var downloadedTrackerDataEtag: String? { get } + var downloadedTrackerData: Data? { get } + +} + +public class TrackerDataManager { + + public enum ReloadResult: Equatable { + case embedded + case embeddedFallback + case downloaded + } + + public typealias DataSet = (tds: TrackerData, etag: String) + + private let lock = NSLock() + + private var _fetchedData: DataSet? + private(set) public var fetchedData: DataSet? { + get { + lock.lock() + let data = _fetchedData + lock.unlock() + return data + } + set { + lock.lock() + _fetchedData = newValue + lock.unlock() + } + } + + private var _embeddedData: DataSet! + private(set) public var embeddedData: DataSet { + get { + lock.lock() + let data: DataSet + // List is loaded lazily when needed + if let embedded = _embeddedData { + data = embedded + } else { + let embedded = embeddedDataProvider.embeddedData + let trackerData = try? JSONDecoder().decode(TrackerData.self, from: embedded) + _embeddedData = (trackerData!, embeddedDataProvider.embeddedDataEtag) + data = _embeddedData + } + lock.unlock() + return data + } + set { + lock.lock() + _embeddedData = newValue + lock.unlock() + } + } + + public var trackerData: TrackerData { + if let data = fetchedData { + return data.tds + } + return embeddedData.tds + } + + private let embeddedDataProvider: EmbeddedDataProvider + private let errorReporting: EventMapping? + + public init(etag: String?, + data: Data?, + embeddedDataProvider: EmbeddedDataProvider, + errorReporting: EventMapping? = nil) { + self.embeddedDataProvider = embeddedDataProvider + self.errorReporting = errorReporting + + reload(etag: etag, data: data) + } + + @discardableResult + public func reload(etag: String?, data: Data?) -> ReloadResult { + + let result: ReloadResult + + if let etag = etag, + let data = data { + result = .downloaded + + do { + // This might fail if the downloaded data is corrupt or format has changed unexpectedly + let data = try JSONDecoder().decode(TrackerData.self, from: data) + fetchedData = (data, etag) + } catch { + errorReporting?.fire(.trackerDataParseFailed, error: error) + fetchedData = nil + return .embeddedFallback + } + } else { + fetchedData = nil + result = .embedded + } + + return result + } +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/TrackerDataQueryExtension.swift b/Sources/BrowserServicesKit/ContentBlocking/TrackerDataQueryExtension.swift new file mode 100644 index 000000000..d42b5f688 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/TrackerDataQueryExtension.swift @@ -0,0 +1,69 @@ +// +// TrackerDataQueryExtension.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import TrackerRadarKit + +extension TrackerData { + + public func findEntity(byName name: String) -> Entity? { + return entities[name] + } + + public func findEntity(forHost host: String) -> Entity? { + for host in variations(of: host) { + if let entityName = domains[host] { + return entities[entityName] + } + } + return nil + } + + private func variations(of host: String) -> [String] { + var parts = host.components(separatedBy: ".") + var domains = [String]() + while parts.count > 1 { + let domain = parts.joined(separator: ".") + domains.append(domain) + parts.removeFirst() + } + return domains + } + + public func findTracker(forUrl url: String) -> KnownTracker? { + guard let host = URL(string: url)?.host else { return nil } + + let variations = variations(of: host) + for host in variations { + if let tracker = trackers[host] { + return tracker + } + } + + for host in variations { + if let cname = cnames?[host] { + var tracker = findTracker(byCname: cname) + tracker = tracker?.copy(withNewDomain: cname) + return tracker + } + } + + return nil + } +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/ContentBlockerRulesUserScript.swift b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/ContentBlockerRulesUserScript.swift new file mode 100644 index 000000000..9cd429257 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/ContentBlockerRulesUserScript.swift @@ -0,0 +1,180 @@ +// +// ContentBlockerRulesUserScript.swift +// DuckDuckGo +// +// Copyright © 2020 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import WebKit +import TrackerRadarKit + +public protocol ContentBlockerRulesUserScriptDelegate: NSObjectProtocol { + + func contentBlockerRulesUserScriptShouldProcessTrackers(_ script: ContentBlockerRulesUserScript) -> Bool + func contentBlockerRulesUserScriptShouldProcessCTLTrackers(_ script: ContentBlockerRulesUserScript) -> Bool + func contentBlockerRulesUserScript(_ script: ContentBlockerRulesUserScript, + detectedTracker tracker: DetectedTracker) + +} + +public protocol ContentBlockerUserScriptConfig: UserScriptSourceProviding { + + var privacyConfiguration: PrivacyConfiguration { get } + var trackerData: TrackerData? { get } + var ctlTrackerData: TrackerData? { get } +} + +public class DefaultContentBlockerUserScriptConfig: ContentBlockerUserScriptConfig { + + public let privacyConfiguration: PrivacyConfiguration + public let trackerData: TrackerData? + public let ctlTrackerData: TrackerData? + + public private(set) var source: String + + public init(privacyConfiguration: PrivacyConfiguration, + trackerData: TrackerData?, // This should be non-optional + ctlTrackerData: TrackerData?, + trackerDataManager: TrackerDataManager? = nil) { + + if trackerData == nil { + // Fallback to embedded + self.trackerData = trackerDataManager?.trackerData + } else { + self.trackerData = trackerData + } + + self.privacyConfiguration = privacyConfiguration + self.ctlTrackerData = ctlTrackerData + + source = ContentBlockerRulesUserScript.generateSource(privacyConfiguration: privacyConfiguration) + } + +} + +open class ContentBlockerRulesUserScript: NSObject, UserScript { + + struct ContentBlockerKey { + static let url = "url" + static let resourceType = "resourceType" + static let blocked = "blocked" + static let pageUrl = "pageUrl" + } + + private let configuration: ContentBlockerUserScriptConfig + + public init(configuration: ContentBlockerUserScriptConfig) { + self.configuration = configuration + + super.init() + } + + public var source: String { + return configuration.source + } + + public var injectionTime: WKUserScriptInjectionTime = .atDocumentStart + + public var forMainFrameOnly: Bool = false + + public var messageNames: [String] = [ "processRule" ] + + public weak var delegate: ContentBlockerRulesUserScriptDelegate? + + var temporaryUnprotectedDomains: [String] { + let privacyConfiguration = configuration.privacyConfiguration + var temporaryUnprotectedDomains = privacyConfiguration.tempUnprotectedDomains.filter { !$0.trimWhitespace().isEmpty } + temporaryUnprotectedDomains.append(contentsOf: privacyConfiguration.exceptionsList(forFeature: .contentBlocking)) + return temporaryUnprotectedDomains + } + + public func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) { + guard let delegate = delegate else { return } + guard delegate.contentBlockerRulesUserScriptShouldProcessTrackers(self) else { return } + let ctlEnabled = delegate.contentBlockerRulesUserScriptShouldProcessCTLTrackers(self) + + guard let dict = message.body as? [String: Any] else { return } + + // False if domain is in unprotected list + guard let blocked = dict[ContentBlockerKey.blocked] as? Bool else { return } + guard let trackerUrlString = dict[ContentBlockerKey.url] as? String else { return } + let resourceType = (dict[ContentBlockerKey.resourceType] as? String) ?? "unknown" + guard let pageUrlStr = dict[ContentBlockerKey.pageUrl] as? String else { return } + + guard let currentTrackerData = configuration.trackerData else { + return + } + + let privacyConfiguration = configuration.privacyConfiguration + + if ctlEnabled, let ctlTrackerData = configuration.ctlTrackerData { + let resolver = TrackerResolver(tds: ctlTrackerData, + unprotectedSites: privacyConfiguration.userUnprotectedDomains, + tempList: temporaryUnprotectedDomains) + + if let tracker = resolver.trackerFromUrl(trackerUrlString, + pageUrlString: pageUrlStr, + resourceType: resourceType, + potentiallyBlocked: blocked && privacyConfiguration.isEnabled(featureKey: .contentBlocking)) { + if tracker.blocked { + delegate.contentBlockerRulesUserScript(self, detectedTracker: tracker) + return + } + } + } + + let resolver = TrackerResolver(tds: currentTrackerData, + unprotectedSites: privacyConfiguration.userUnprotectedDomains, + tempList: temporaryUnprotectedDomains) + + if let tracker = resolver.trackerFromUrl(trackerUrlString, + pageUrlString: pageUrlStr, + resourceType: resourceType, + potentiallyBlocked: blocked && privacyConfiguration.isEnabled(featureKey: .contentBlocking)) { + delegate.contentBlockerRulesUserScript(self, detectedTracker: tracker) + } + } + + public static func generateSource(privacyConfiguration: PrivacyConfiguration) -> String { + let remoteUnprotectedDomains = (privacyConfiguration.tempUnprotectedDomains.joined(separator: "\n")) + + "\n" + + (privacyConfiguration.exceptionsList(forFeature: .contentBlocking).joined(separator: "\n")) + + return ContentBlockerRulesUserScript.loadJS("contentblockerrules", from: Bundle.module, withReplacements: [ + "$TEMP_UNPROTECTED_DOMAINS$": remoteUnprotectedDomains, + "$USER_UNPROTECTED_DOMAINS$": privacyConfiguration.userUnprotectedDomains.joined(separator: "\n"), + "$TRACKER_ALLOWLIST_ENTRIES$": TrackerAllowlistInjection.prepareForInjection(allowlist: privacyConfiguration.trackerAllowlist) + ]) + } +} + +public class TrackerAllowlistInjection { + + static public func prepareForInjection(allowlist: PrivacyConfigurationData.TrackerAllowlistData) -> String { + // Transform rules into regular expresions + var output = PrivacyConfigurationData.TrackerAllowlistData() + for dictEntry in allowlist { + let newValue = dictEntry.value.map { entry -> PrivacyConfigurationData.TrackerAllowlist.Entry in + let regexp = ContentBlockerRulesBuilder.makeRegexpFilter(fromAllowlistRule: entry.rule) + let escapedRegexp = regexp.replacingOccurrences(of: "\\", with: "\\\\") + return PrivacyConfigurationData.TrackerAllowlist.Entry(rule: escapedRegexp, domains: entry.domains) + } + output[dictEntry.key] = newValue + } + + return (try? String(data: JSONEncoder().encode(output), encoding: .utf8)) ?? "" + } + +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift new file mode 100644 index 000000000..9431e0def --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift @@ -0,0 +1,162 @@ +// +// SurrogatesUserScript.swift +// Core +// +// Copyright © 2020 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import WebKit +import os +import TrackerRadarKit + +public protocol SurrogatesUserScriptDelegate: NSObjectProtocol { + + func surrogatesUserScriptShouldProcessTrackers(_ script: SurrogatesUserScript) -> Bool + func surrogatesUserScript(_ script: SurrogatesUserScript, + detectedTracker tracker: DetectedTracker, + withSurrogate host: String) + +} + +public protocol SurrogatesUserScriptConfig: UserScriptSourceProviding { + + var privacyConfig: PrivacyConfiguration { get } + var surrogates: String { get } + var trackerData: TrackerData? { get } + var encodedSurrogateTrackerData: String? { get } + +} + +public class DefaultSurrogatesUserScriptConfig: SurrogatesUserScriptConfig { + + public let privacyConfig: PrivacyConfiguration + public let surrogates: String + public let trackerData: TrackerData? + public let encodedSurrogateTrackerData: String? + + public let source: String + + public init(privacyConfig: PrivacyConfiguration, + surrogates: String, + trackerData: TrackerData?, + encodedSurrogateTrackerData: String?, + trackerDataManager: TrackerDataManager, + isDebugBuild: Bool) { + + if trackerData == nil { + // Fallback to embedded + self.trackerData = trackerDataManager.trackerData + + let surrogateTDS = ContentBlockerRulesManager.extractSurrogates(from: trackerDataManager.trackerData) + let encodedData = try? JSONEncoder().encode(surrogateTDS) + let encodedTrackerData = String(data: encodedData!, encoding: .utf8)! + self.encodedSurrogateTrackerData = encodedTrackerData + } else { + self.trackerData = trackerData + self.encodedSurrogateTrackerData = encodedSurrogateTrackerData + } + + self.privacyConfig = privacyConfig + self.surrogates = surrogates + + source = SurrogatesUserScript.generateSource(privacyConfiguration: self.privacyConfig, + surrogates: self.surrogates, + encodedSurrogateTrackerData: self.encodedSurrogateTrackerData, + isDebugBuild: isDebugBuild) + } +} + +open class SurrogatesUserScript: NSObject, UserScript { + + struct TrackerDetectedKey { + static let protectionId = "protectionId" + static let blocked = "blocked" + static let networkName = "networkName" + static let url = "url" + static let isSurrogate = "isSurrogate" + static let pageUrl = "pageUrl" + } + + private let configuration: SurrogatesUserScriptConfig + + public init(configuration: SurrogatesUserScriptConfig) { + self.configuration = configuration + + super.init() + } + + open var source: String { + return configuration.source + } + + public var injectionTime: WKUserScriptInjectionTime = .atDocumentStart + + public var forMainFrameOnly: Bool = false + + public var messageNames: [String] = [ "trackerDetectedMessage" ] + + public weak var delegate: SurrogatesUserScriptDelegate? + + public func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) { + guard let delegate = delegate else { return } + guard delegate.surrogatesUserScriptShouldProcessTrackers(self) else { return } + + guard let dict = message.body as? [String: Any] else { return } + guard let blocked = dict[TrackerDetectedKey.blocked] as? Bool else { return } + guard let urlString = dict[TrackerDetectedKey.url] as? String else { return } + guard let pageUrlStr = dict[TrackerDetectedKey.pageUrl] as? String else { return } + + let tracker = trackerFromUrl(urlString.trimWhitespace(), pageUrlString: pageUrlStr, blocked) + + if let isSurrogate = dict[TrackerDetectedKey.isSurrogate] as? Bool, isSurrogate, let host = URL(string: urlString)?.host { + delegate.surrogatesUserScript(self, detectedTracker: tracker, withSurrogate: host) + } + } + + private func trackerFromUrl(_ urlString: String, pageUrlString: String, _ blocked: Bool) -> DetectedTracker { + let currentTrackerData = configuration.trackerData + let knownTracker = currentTrackerData?.findTracker(forUrl: urlString) + let entity = currentTrackerData?.findEntity(byName: knownTracker?.owner?.name ?? "") + return DetectedTracker(url: urlString, knownTracker: knownTracker, entity: entity, blocked: blocked, pageUrl: pageUrlString) + } + + public static func generateSource(privacyConfiguration: PrivacyConfiguration, + surrogates: String, + encodedSurrogateTrackerData: String?, + isDebugBuild: Bool) -> String { + let remoteUnprotectedDomains = (privacyConfiguration.tempUnprotectedDomains.joined(separator: "\n")) + + "\n" + + (privacyConfiguration.exceptionsList(forFeature: .contentBlocking).joined(separator: "\n")) + + // Encode whatever the tracker data manager is using to ensure it's in sync and because we know it will work + let trackerData: String + if let data = encodedSurrogateTrackerData { + trackerData = data + } else { + let encodedData = try? JSONEncoder().encode(TrackerData(trackers: [:], entities: [:], domains: [:], cnames: [:])) + trackerData = String(data: encodedData!, encoding: .utf8)! + } + + return SurrogatesUserScript.loadJS("surrogates", from: Bundle.module, withReplacements: [ + "$IS_DEBUG$": isDebugBuild ? "true" : "false", + "$TEMP_UNPROTECTED_DOMAINS$": remoteUnprotectedDomains, + "$USER_UNPROTECTED_DOMAINS$": privacyConfiguration.userUnprotectedDomains.joined(separator: "\n"), + "$TRACKER_ALLOWLIST_ENTRIES$": TrackerAllowlistInjection.prepareForInjection(allowlist: privacyConfiguration.trackerAllowlist), + "$TRACKER_DATA$": trackerData, + "$SURROGATES$": surrogates, + "$BLOCKING_ENABLED$": privacyConfiguration.isEnabled(featureKey: .contentBlocking) ? "true" : "false" + ]) + } +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift new file mode 100644 index 000000000..c0d65aff1 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift @@ -0,0 +1,143 @@ +// +// TrackerResolver.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import TrackerRadarKit + +public class TrackerResolver { + + let tds: TrackerData + let unprotectedSites: [String] + let tempList: [String] + + public init(tds: TrackerData, unprotectedSites: [String], tempList: [String]) { + self.tds = tds + self.unprotectedSites = unprotectedSites + self.tempList = tempList + } + + public func trackerFromUrl(_ trackerUrlString: String, + pageUrlString: String, + resourceType: String, + potentiallyBlocked: Bool) -> DetectedTracker? { + + guard let tracker = tds.findTracker(forUrl: trackerUrlString), + let entity = tds.findEntity(byName: tracker.owner?.name ?? "") else { + return nil + } + + let blocked: Bool + + // Check for unprotected domains + if let pageDomain = URL(string: pageUrlString), + let pageHost = pageDomain.host, + unprotectedSites.contains(pageHost) || tempList.contains(pageHost) { + blocked = false + } else { + // Check for custom rules + let rule = tracker.hasRule(for: trackerUrlString, type: resourceType, pageUrlString: pageUrlString) + switch rule { + case .none: + if tracker.defaultAction == .block { + blocked = potentiallyBlocked + } else /* if tracker.defaultAction == .ignore */ { + blocked = false + } + case .allowRequest: + blocked = false + case .blockRequest: + blocked = potentiallyBlocked + } + } + + // Make sure current page is not affilated with the tracker + if let pageUrl = URL(string: pageUrlString), + let pageHost = pageUrl.host, + let pageEntity = tds.findEntity(forHost: pageHost), + pageEntity.displayName == entity.displayName { + return nil + } + + return DetectedTracker(url: trackerUrlString, knownTracker: tracker, entity: entity, blocked: blocked, pageUrl: pageUrlString) + } + + enum RuleAction { + case none + case allowRequest + case blockRequest + } + + static public func isMatching(_ option: KnownTracker.Rule.Matching, host: String, resourceType: String) -> Bool { + + var isEmpty = true // Require either domains or types to be specified + var matching = true + + if let requiredDomains = option.domains, !requiredDomains.isEmpty { + isEmpty = false + matching = requiredDomains.contains(where: { domain in + guard domain != host else { return true } + return host.hasSuffix(".\(domain)") + }) + } + + if let requiredTypes = option.types, !requiredTypes.isEmpty { + isEmpty = false + matching = matching && requiredTypes.contains(resourceType) + } + + return !isEmpty && matching + } +} + +fileprivate extension KnownTracker { + + func hasRule(for trackerUrlString: String, + type: String, + pageUrlString: String) -> TrackerResolver.RuleAction { + + let range = NSRange(location: 0, length: trackerUrlString.utf16.count) + let host = URL(string: pageUrlString)?.host + + for rule in rules ?? [] { + guard let pattern = rule.rule, + let host = host, + let regex = try? NSRegularExpression(pattern: pattern, options: []) else { continue } + + if regex.firstMatch(in: trackerUrlString, options: [], range: range) != nil { + + // If rule is set to 'ignore', we note it is tracker but allow the request. + if rule.action == .ignore { + return .allowRequest + } + + if let options = rule.options, !TrackerResolver.isMatching(options, host: host, resourceType: type) { + return .allowRequest + } + + if let exceptions = rule.exceptions, TrackerResolver.isMatching(exceptions, host: host, resourceType: type) { + return .allowRequest + } + + return .blockRequest + } + } + + return .none + } +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/contentblockerrules.js b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/contentblockerrules.js new file mode 100644 index 000000000..1833a03e3 --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/contentblockerrules.js @@ -0,0 +1,334 @@ +// +// contentblockerrules.js +// DuckDuckGo +// +// Copyright © 2020 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// "use strict"; + +(function () { + const topLevelUrl = getTopLevelURL() + + let unprotectedDomain = false + const domainParts = topLevelUrl && topLevelUrl.host ? topLevelUrl.host.split('.') : [] + + // walk up the domain to see if it's unprotected + while (domainParts.length > 1 && !unprotectedDomain) { + const partialDomain = domainParts.join('.') + + unprotectedDomain = ` + $TEMP_UNPROTECTED_DOMAINS$ + `.split('\n').filter(domain => domain.trim() === partialDomain).length > 0 + + domainParts.shift() + } + + if (!unprotectedDomain && topLevelUrl.host != null && topLevelUrl.host.length > 0) { + unprotectedDomain = ` + $USER_UNPROTECTED_DOMAINS$ + `.split('\n').filter(domain => domain.trim() === topLevelUrl.host).length > 0 + } + + // tld.js + const tldjs = { + + parse: function (url) { + if (url.startsWith('//')) { + url = 'http:' + url + } + + try { + const parsed = new URL(url) + return { + domain: parsed.hostname, + hostname: parsed.hostname + } + } catch (error) { + return { + domain: '', + hostname: '' + } + } + } + + } + // tld.js + + let trackerAllowlist = {} + const trackerAllowlistEntries = ` + $TRACKER_ALLOWLIST_ENTRIES$ + ` + + if (trackerAllowlistEntries) { + trackerAllowlist = JSON.parse(trackerAllowlistEntries) + } + + function isTrackerAllowlisted (siteURL, request) { + // check that allowlist has entries + if (!Object.keys(trackerAllowlist).length) { + return false + } + + const parsedRequest = tldjs.parse(request) + const requestDomainParts = Array.from(parsedRequest.domain.split('.')) + + let allowListEntry = null + while (requestDomainParts.length > 1) { + const requestDomain = requestDomainParts.join('.') + + allowListEntry = trackerAllowlist[requestDomain] + if (allowListEntry) { + break + } + requestDomainParts.shift() + } + + if (allowListEntry) { + return _matchesRule(siteURL, request, allowListEntry) + } else { + return false + } + } + + function _matchesRule (siteURL, request, allowListEntryList) { + let matchedEntry = null + + if (allowListEntryList && allowListEntryList.length) { + for (const entryObj of allowListEntryList) { + if (request.match(entryObj.rule)) { + matchedEntry = entryObj + break + } + } + } + + if (matchedEntry) { + if (matchedEntry.domains.includes('')) { + return true + } + + const siteDomainParts = Array.from(siteURL.host.split('.')) + + while (siteDomainParts.length > 1) { + const siteDomain = siteDomainParts.join('.') + if (matchedEntry.domains.includes(siteDomain)) { + return true + } + siteDomainParts.shift() + } + } + + return false + } + + // private + function getTopLevelURL () { + try { + // FROM: https://stackoverflow.com/a/7739035/73479 + // FIX: Better capturing of top level URL so that trackers in embedded documents are not considered first party + if (window.location !== window.parent.location) { + return new URL(window.location.href !== 'about:blank' ? document.referrer : window.parent.location.href) + } else { + return new URL(document.location.href) + } + } catch (error) { + return new URL(location.href) + } + } + + if (!window.__firefox__) { + Object.defineProperty(window, '__firefox__', { + enumerable: false, + configurable: false, + writable: false, + value: { + userScripts: {}, + includeOnce: function (userScript, initializer) { + if (!__firefox__.userScripts[userScript]) { + __firefox__.userScripts[userScript] = true + if (typeof initializer === 'function') { + initializer() + } + return false + } + return true + } + } + }) + } + + if (webkit.messageHandlers.processRule) { + install() + } + + function install () { + function sendMessage (url, resourceType) { + if (url) { + webkit.messageHandlers.processRule.postMessage({ + url: url, + resourceType: resourceType === undefined ? null : resourceType, + blocked: !unprotectedDomain && !isTrackerAllowlisted(topLevelUrl, url), + pageUrl: topLevelUrl.href + }) + } + } + + function onLoadNativeCallback () { + // Send back the sources of every script and image in the DOM back to the host application. + [].slice.apply(document.scripts).forEach(function (el) { sendMessage(el.src, 'script') }); + [].slice.apply(document.querySelectorAll('link')).forEach(function (el) { sendMessage(el.href, 'link') }); + [].slice.apply(document.images).forEach(function (el) { + // If the image's natural width is zero, then it has not loaded so we + // can assume that it may have been blocked. + if (el.naturalWidth === 0) { + sendMessage(el.src, 'image') + } + }); + [].slice.apply(document.querySelectorAll('iframe')).forEach(function (el) { sendMessage(el.src, 'iframe') }) + } + + let originalOpen = null + let originalSend = null + let originalImageSrc = null + let originalFetch = null + let mutationObserver = null + + function injectStatsTracking (enabled) { + // This enable/disable section is a change from the original Focus iOS version. + if (enabled) { + if (originalOpen) { + return + } + window.addEventListener('load', onLoadNativeCallback, false) + } else { + window.removeEventListener('load', onLoadNativeCallback, false) + + if (originalOpen) { // if one is set, then all the enable code has run + XMLHttpRequest.prototype.open = originalOpen + XMLHttpRequest.prototype.send = originalSend + Image.prototype.src = originalImageSrc + mutationObserver.disconnect() + + originalOpen = originalSend = originalImageSrc = mutationObserver = null + } + return + } + + // ------------------------------------------------- + // Send ajax requests URLs to the host application + // ------------------------------------------------- + const xhrProto = XMLHttpRequest.prototype + if (!originalOpen) { + originalOpen = xhrProto.open + originalSend = xhrProto.send + } + + xhrProto.open = function (method, url) { + this._url = url + return originalOpen.apply(this, arguments) + } + + xhrProto.send = function (body) { + // Only attach the `error` event listener once for this + // `XMLHttpRequest` instance. + if (!this._tpErrorHandler) { + // If this `XMLHttpRequest` instance fails to load, we + // can assume it has been blocked. + this._tpErrorHandler = function () { + sendMessage(this._url, 'xmlhttprequest') + } + this.addEventListener('error', this._tpErrorHandler) + } + return originalSend.apply(this, arguments) + } + + // ------------------------------------------------- + // Detect when new sources get set on Image and send them to the host application + // ------------------------------------------------- + if (!originalImageSrc) { + originalImageSrc = Object.getOwnPropertyDescriptor(Image.prototype, 'src') + } + + delete Image.prototype.src + Object.defineProperty(Image.prototype, 'src', { + configurable: true, + get: function () { + return originalImageSrc.get.call(this) + }, + set: function (value) { + // Only attach the `error` event listener once for this + // Image instance. + if (!this._tpErrorHandler) { + // If this `Image` instance fails to load, we can assume + // it has been blocked. + this._tpErrorHandler = function () { + sendMessage(this.src, 'image') + } + this.addEventListener('error', this._tpErrorHandler) + } + + originalImageSrc.set.call(this, value) + } + }) + + // ------------------------------------------------- + // Detect when fetch is called and pass the resource to the host application + // ------------------------------------------------- + if (!originalFetch) { + originalFetch = window.fetch + } + window.fetch = function () { + if (arguments.length === 0) { + return originalFetch.apply(window, arguments) + } + + if (typeof arguments[0] === 'string') { + sendMessage(arguments[0], 'fetch') + } else if (arguments[0].url) { + // Argument is a Request object + sendMessage(arguments[0].url, 'fetch') + } + + return originalFetch.apply(window, arguments) + } + + // ------------------------------------------------- + // Listen to when new " + } + } + + return """ + + + +

Test Page

+ \(content) + + + """ + } +} diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift new file mode 100644 index 000000000..8b963f086 --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift @@ -0,0 +1,529 @@ +// +// SurrogatesUserScriptTests.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import WebKit +import BrowserServicesKit +import TrackerRadarKit + +// swiftlint:disable file_length +// swiftlint:disable type_body_length +class SurrogatesUserScriptsTests: XCTestCase { + + static let exampleRules = """ +{ + "trackers": { + "tracker.com": { + "domain": "tracker.com", + "default": "block", + "rules": [ + { + "rule": "tracker\\\\.com\\\\/scripts\\\\/script\\\\.js", + "surrogate": "script.js" + } + ], + "owner": { + "name": "Fake Tracking Inc", + "displayName": "FT Inc", + "privacyPolicy": "https://tracker.com/privacy", + "url": "http://tracker.com" + }, + "source": [ + "DDG" + ], + "prevalence": 0.002, + "fingerprinting": 0, + "cookies": 0.002, + "performance": { + "time": 1, + "size": 1, + "cpu": 1, + "cache": 3 + }, + "categories": [ + "Ad Motivated Tracking", + "Advertising", + "Analytics", + "Third-Party Analytics Marketing" + ] + } + }, + "entities": { + "Fake Tracking Inc": { + "domains": [ + "tracker.com" + ], + "displayName": "Fake Tracking Inc", + "prevalence": 0.1 + } + }, + "domains": { + "tracker.com": "Fake Tracking Inc" + } +} +""" + + static let exampleSurrogates = """ + tracker.com/script.js application/javascript + (() => { + 'use strict'; + var surrogatesScriptTest = function() { + function ping() { + return "success" + } + return { + ping: ping + } + }() + window.surrT = surrogatesScriptTest + })(); + """ + + let schemeHandler = TestSchemeHandler() + let userScriptDelegateMock = MockSurrogatesUserScriptDelegate() + let navigationDelegateMock = MockNavigationDelegate() + + var webView: WKWebView? + + let nonTrackerURL = URL(string: "test://nontracker.com/1.png")! + let trackerURL = URL(string: "test://tracker.com/1.png")! + let surrogateScriptURL = URL(string: "test://tracker.com/scripts/script.js")! + let nonSurrogateScriptURL = URL(string: "test://tracker.com/other/script.js")! + + var website: MockWebsite! + + override func setUp() { + super.setUp() + + website = MockWebsite(resources: [.init(type: .image, url: nonTrackerURL), + .init(type: .image, url: trackerURL), + .init(type: .script, url: surrogateScriptURL), + .init(type: .script, url: nonSurrogateScriptURL)]) + } + + private func setupWebViewForUserScripTests(trackerData: TrackerData, + encodedTrackerData: String, + privacyConfig: PrivacyConfiguration, + completion: @escaping (WKWebView) -> Void) { + + var tempUnprotected = privacyConfig.tempUnprotectedDomains.filter { !$0.trimWhitespace().isEmpty } + tempUnprotected.append(contentsOf: privacyConfig.exceptionsList(forFeature: .contentBlocking)) + + let exceptions = DefaultContentBlockerRulesExceptionsSource.transform(allowList: privacyConfig.trackerAllowlist) + + WebKitTestHelper.prepareContentBlockingRules(trackerData: trackerData, + exceptions: privacyConfig.userUnprotectedDomains, + tempUnprotected: tempUnprotected, + trackerExceptions: exceptions) { rules in + guard let rules = rules else { + XCTFail("Rules were not compiled properly") + return + } + + let configuration = WKWebViewConfiguration() + configuration.setURLSchemeHandler(self.schemeHandler, forURLScheme: self.schemeHandler.scheme) + + let webView = WKWebView(frame: .init(origin: .zero, size: .init(width: 500, height: 1000)), + configuration: configuration) + webView.navigationDelegate = self.navigationDelegateMock + + let config = TestSchemeSurrogatesUserScriptConfig(privacyConfig: privacyConfig, + surrogates: Self.exampleSurrogates, + trackerData: trackerData, + encodedSurrogateTrackerData: encodedTrackerData, + isDebugBuild: true) + + let userScript = SurrogatesUserScript(configuration: config) + userScript.delegate = self.userScriptDelegateMock + + for messageName in userScript.messageNames { + configuration.userContentController.add(userScript, name: messageName) + } + + configuration.userContentController.addUserScript(WKUserScript(source: userScript.source, + injectionTime: .atDocumentStart, + forMainFrameOnly: false)) + configuration.userContentController.add(rules) + + completion(webView) + } + } + + private func performTestFor(privacyConfig: PrivacyConfiguration, + websiteURL: URL) { + + let trackerDataSource = Self.exampleRules.data(using: .utf8)! + let trackerData = (try? JSONDecoder().decode(TrackerData.self, from: trackerDataSource))! + + let encodedData = try? JSONEncoder().encode(trackerData) + let encodedTrackerData = String(data: encodedData!, encoding: .utf8)! + + setupWebViewForUserScripTests(trackerData: trackerData, + encodedTrackerData: encodedTrackerData, + privacyConfig: privacyConfig) { webView in + // Keep webview in memory till test finishes + self.webView = webView + + self.schemeHandler.requestHandlers[websiteURL] = { _ in + return self.website.htmlRepresentation.data(using: .utf8)! + } + + let request = URLRequest(url: websiteURL) + WKWebsiteDataStore.default().removeData(ofTypes: [WKWebsiteDataTypeDiskCache, + WKWebsiteDataTypeMemoryCache], + modifiedSince: Date(timeIntervalSince1970: 0), + completionHandler: { + webView.load(request) + }) + } + } + + func testWhenThereIsSurrogateRuleThenSurrogateIsInjected() { + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + let websiteURL = URL(string: "test://example.com")! + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 1) + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.first?.url, self.surrogateScriptURL.absoluteString) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { result, err in + XCTAssertNil(err) + if let result = result as? String { + XCTAssertEqual(result, "success") + surrogateValidated.fulfill() + } + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenSiteIsLocallyUnprotectedThenSurrogatesAreNotInjected() { + + let websiteURL = URL(string: "test://example.com")! + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: ["example.com"], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNotNil(err) + surrogateValidated.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenSiteIsSubdomainOfLocallyUnprotectedThenSurrogatesAreInjected() { + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: ["example.com"], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + + let websiteURL = URL(string: "test://sub.example.com")! + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 1) + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.first?.url, self.surrogateScriptURL.absoluteString) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { result, err in + XCTAssertNil(err) + if let result = result as? String { + XCTAssertEqual(result, "success") + surrogateValidated.fulfill() + } + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenSiteIsTempUnprotectedThenSurrogatesAreNotInjected() { + + let websiteURL = URL(string: "test://example.com")! + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: ["example.com"], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNotNil(err) + surrogateValidated.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenSiteIsSubdomainOfTempUnprotectedThenSurrogatesAreNotInjected() { + + let websiteURL = URL(string: "test://sub.example.com")! + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: ["example.com"], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNotNil(err) + surrogateValidated.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenSiteIsInExceptionListThenSurrogatesAreNotInjected() { + + let websiteURL = URL(string: "test://example.com")! + + let allowlist = ["tracker.com": [PrivacyConfigurationData.TrackerAllowlist.Entry(rule: "tracker.com/", domains: ["example.com"])]] + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: allowlist, + contentBlockingEnabled: true, + exceptions: []) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNotNil(err) + surrogateValidated.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenSiteIsNotInExceptionListThenSurrogatesAreInjected() { + + let websiteURL = URL(string: "test://example.com")! + + let allowlist = ["tracker.com": [PrivacyConfigurationData.TrackerAllowlist.Entry(rule: "tracker.com/", domains: ["test.com"])]] + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: allowlist, + contentBlockingEnabled: true, + exceptions: []) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 1) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNil(err) + surrogateValidated.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenTrackerIsInAllowListThenSurrogatesAreNotInjected() { + + let websiteURL = URL(string: "test://example.com")! + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: ["example.com"]) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNotNil(err) + surrogateValidated.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenSiteIsSubdomainOfExceptionListThenSurrogatesAreNotInjected() { + + let websiteURL = URL(string: "test://sub.example.com")! + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: ["example.com"]) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNotNil(err) + surrogateValidated.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenContentBlockingFeatureIsDisabledThenSurrogatesAreNotInjected() { + + let websiteURL = URL(string: "test://sub.example.com")! + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: false, + exceptions: []) + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + websiteLoaded.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + + self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in + XCTAssertNotNil(err) + surrogateValidated.fulfill() + }) + + // Note: do not check the requests - they will be blocked as test setup adds content blocking rules + // despite feature flag being set to false - so we validate only how Surrogates script handles that. + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } +} +// swiftlint:enable type_body_length +// swiftlint:enable file_length diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/TestSchemeHandler.swift b/Tests/BrowserServicesKitTests/ContentBlocker/TestSchemeHandler.swift new file mode 100644 index 000000000..ffa364ba3 --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/TestSchemeHandler.swift @@ -0,0 +1,59 @@ +// +// TestSchemeHandler.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import WebKit + +final class TestSchemeHandler: NSObject, WKURLSchemeHandler { + typealias RequestResponse = (URL) -> Data + + public var requestHandlers = [URL: RequestResponse]() + + public let scheme = "test" + + public var genericHandler: RequestResponse = { _ in + return Data() + } + + public var handledRequests = [URL]() + + func reset() { + requestHandlers.removeAll() + handledRequests.removeAll() + } + + func webView(_ webView: WKWebView, stop urlSchemeTask: WKURLSchemeTask) { } + + func webView(_ webView: WKWebView, start urlSchemeTask: WKURLSchemeTask) { + let url = urlSchemeTask.request.url! + handledRequests.append(url) + + let handler = self.requestHandlers[url] ?? self.genericHandler + + let data = handler(url) + + let response = URLResponse(url: url, + mimeType: "text/html", + expectedContentLength: data.count, + textEncodingName: nil) + urlSchemeTask.didReceive(response) + urlSchemeTask.didReceive(data) + urlSchemeTask.didFinish() + } +} diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/TrackerAllowlistReferenceTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerAllowlistReferenceTests.swift new file mode 100644 index 000000000..e00f43ff2 --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerAllowlistReferenceTests.swift @@ -0,0 +1,216 @@ +// +// TrackerAllowlistReferenceTests.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import os.log +import WebKit +import BrowserServicesKit +import TrackerRadarKit + +struct AllowlistTests: Decodable { + + struct Test: Decodable { + + let description: String + let site: String + let request: String + let isAllowlisted: Bool + + } + + let domainTests: [Test] +} + +class TrackerAllowlistReferenceTests: XCTestCase { + + let schemeHandler = TestSchemeHandler() + let userScriptDelegateMock = MockRulesUserScriptDelegate() + let navigationDelegateMock = MockNavigationDelegate() + + var webView: WKWebView! + var tds: TrackerData! + var tests = [AllowlistTests.Test]() + var mockWebsite: MockWebsite! + + override func setUp() { + super.setUp() + } + + func setupWebView(trackerData: TrackerData, + userScriptDelegate: ContentBlockerRulesUserScriptDelegate, + trackerAllowlist: PrivacyConfigurationData.TrackerAllowlistData, + schemeHandler: TestSchemeHandler, + completion: @escaping (WKWebView) -> Void) { + + let exceptions = DefaultContentBlockerRulesExceptionsSource.transform(allowList: trackerAllowlist) + + WebKitTestHelper.prepareContentBlockingRules(trackerData: trackerData, + exceptions: [], + tempUnprotected: [], + trackerExceptions: exceptions) { rules in + guard let rules = rules else { + XCTFail("Rules were not compiled properly") + return + } + + let configuration = WKWebViewConfiguration() + configuration.setURLSchemeHandler(schemeHandler, forURLScheme: schemeHandler.scheme) + + let webView = WKWebView(frame: .init(origin: .zero, size: .init(width: 500, height: 1000)), + configuration: configuration) + webView.navigationDelegate = self.navigationDelegateMock + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: trackerAllowlist, + contentBlockingEnabled: true, + exceptions: []) + + let config = TestSchemeContentBlockerUserScriptConfig(privacyConfiguration: privacyConfig, + trackerData: trackerData, + ctlTrackerData: nil) + + let userScript = ContentBlockerRulesUserScript(configuration: config) + userScript.delegate = userScriptDelegate + + for messageName in userScript.messageNames { + configuration.userContentController.add(userScript, name: messageName) + } + + configuration.userContentController.addUserScript(WKUserScript(source: userScript.source, + injectionTime: .atDocumentStart, + forMainFrameOnly: false)) + configuration.userContentController.add(rules) + + completion(webView) + } + } + + func testDomainAllowlist() throws { + + let data = JsonTestDataLoader() + let trackerJSON = data.fromJsonFile("privacy-reference-tests/tracker-radar-tests/TR-domain-matching/tracker_allowlist_tds_reference.json") + let testJSON = data.fromJsonFile("privacy-reference-tests/tracker-radar-tests/TR-domain-matching/tracker_allowlist_matching_tests.json") + + let allowlistReference = data.fromJsonFile("privacy-reference-tests/tracker-radar-tests/TR-domain-matching/tracker_allowlist_reference.json") + + tds = try JSONDecoder().decode(TrackerData.self, from: trackerJSON) + + let allowlistJson = try? JSONSerialization.jsonObject(with: allowlistReference, options: []) as? [String: Any] + + let allowlist = PrivacyConfigurationData.TrackerAllowlist(json: ["state": "enabled", "settings": ["allowlistedTrackers": allowlistJson]])! + + let refTests = try JSONDecoder().decode(Array.self, from: testJSON) + tests = refTests + + let testsExecuted = expectation(description: "tests executed") + testsExecuted.expectedFulfillmentCount = tests.count + + setupWebView(trackerData: tds, + userScriptDelegate: userScriptDelegateMock, + trackerAllowlist: allowlist.entries, + schemeHandler: schemeHandler) { webView in + self.webView = webView + + self.popTestAndExecute(onTestExecuted: testsExecuted) + } + + waitForExpectations(timeout: 30, handler: nil) + } + + private func normalizeScheme(urlString: String) -> String { + return urlString.replacingOccurrences(of: "https://", with: "test://").replacingOccurrences(of: "http://", with: "test://") + } + + // swiftlint:disable function_body_length + private func popTestAndExecute(onTestExecuted: XCTestExpectation) { + + guard let test = tests.popLast() else { + return + } + + os_log("TEST: %s", test.description) + + let siteURL = URL(string: normalizeScheme(urlString: test.site))! + let requestURL = URL(string: normalizeScheme(urlString: test.request))! + + let resource = MockWebsite.EmbeddedResource(type: .script, + url: requestURL) + + mockWebsite = MockWebsite(resources: [resource]) + + schemeHandler.reset() + schemeHandler.requestHandlers[siteURL] = { _ in + return self.mockWebsite.htmlRepresentation.data(using: .utf8)! + } + + userScriptDelegateMock.reset() + + os_log("Loading %s ...", siteURL.absoluteString) + let request = URLRequest(url: siteURL) + + WKWebsiteDataStore.default().removeData(ofTypes: [WKWebsiteDataTypeDiskCache, + WKWebsiteDataTypeMemoryCache], + modifiedSince: Date(timeIntervalSince1970: 0), + completionHandler: { + self.webView.load(request) + }) + + navigationDelegateMock.onDidFinishNavigation = { + os_log("Website loaded") + if !test.isAllowlisted { + // Only website request + XCTAssertEqual(self.schemeHandler.handledRequests.count, 1) + // Only resource request + XCTAssertEqual(self.userScriptDelegateMock.detectedTrackers.count, 1) + + if let tracker = self.userScriptDelegateMock.detectedTrackers.first { + XCTAssert(tracker.blocked) + } else { + XCTFail("Expected to detect tracker for test \(test.description)") + } + } else { + // Website request & resource request + XCTAssertEqual(self.schemeHandler.handledRequests.count, 2) + + if let pageEntity = self.tds.findEntity(forHost: siteURL.host!), + let trackerOwner = self.tds.findTracker(forUrl: requestURL.absoluteString)?.owner, + pageEntity.displayName == trackerOwner.name { + + // Nothing to detect - tracker and website have the same entity + } else { + XCTAssertEqual(self.userScriptDelegateMock.detectedTrackers.count, 1) + + if let tracker = self.userScriptDelegateMock.detectedTrackers.first { + XCTAssertFalse(tracker.blocked) + } else { + XCTFail("Expected to detect tracker for test \(test.description)") + } + } + } + + onTestExecuted.fulfill() + DispatchQueue.main.async { + self.popTestAndExecute(onTestExecuted: onTestExecuted) + } + } + } + // swiftlint:enable function_body_length + +} diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/TrackerDataManagerTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerDataManagerTests.swift new file mode 100644 index 000000000..5953fcb54 --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerDataManagerTests.swift @@ -0,0 +1,149 @@ +// +// TrackerDataManagerTests.swift +// Core +// +// Copyright © 2019 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import CommonCrypto +import TrackerRadarKit +@testable import BrowserServicesKit +import WebKit + +class TrackerDataManagerTests: XCTestCase { + + static let exampleTDS = """ + { + "trackers": { + "notreal.io": { + "domain": "notreal.io", + "default": "block", + "owner": { + "name": "Not Real LLC", + "displayName": "Not Real", + "privacyPolicy": "https://hermann.ai/privacy-en", + "url": "http://hermann.ai" + }, + "source": [ + "DDG" + ], + "prevalence": 0.002, + "fingerprinting": 0, + "cookies": 0.002, + "performance": { + "time": 1, + "size": 1, + "cpu": 1, + "cache": 3 + }, + "categories": [ + "Ad Motivated Tracking", + "Advertising", + "Analytics", + "Third-Party Analytics Marketing" + ] + } + }, + "entities": { + "Not Real": { + "domains": [ + "notreal.io" + ], + "displayName": "Not Real", + "prevalence": 0.666 + } + }, + "domains": { + "notreal.io": "Not Real" + } + } + """ + + func testWhenReloadCalledInitiallyThenDataSetIsEmbedded() { + + let exampleData = Self.exampleTDS.data(using: .utf8)! + let embeddedDataProvider = MockEmbeddedDataProvider(data: exampleData, + etag: "embedded") + + XCTAssertEqual(TrackerDataManager(etag: nil, + data: nil, + embeddedDataProvider: embeddedDataProvider).reload(etag: nil, + data: nil), + TrackerDataManager.ReloadResult.embedded) + } + + func testFindTrackerByUrl() { + let exampleData = Self.exampleTDS.data(using: .utf8)! + let embeddedDataProvider = MockEmbeddedDataProvider(data: exampleData, + etag: "embedded") + + let trackerDataManager = TrackerDataManager(etag: nil, + data: nil, + embeddedDataProvider: embeddedDataProvider) + let tracker = trackerDataManager.embeddedData.tds.findTracker(forUrl: "http://notreal.io") + XCTAssertNotNil(tracker) + XCTAssertEqual("Not Real", tracker?.owner?.displayName) + } + + func testFindEntityByName() { + let exampleData = Self.exampleTDS.data(using: .utf8)! + let embeddedDataProvider = MockEmbeddedDataProvider(data: exampleData, + etag: "embedded") + + let trackerDataManager = TrackerDataManager(etag: nil, + data: nil, + embeddedDataProvider: embeddedDataProvider) + let entity = trackerDataManager.embeddedData.tds.findEntity(byName: "Not Real") + XCTAssertNotNil(entity) + XCTAssertEqual("Not Real", entity?.displayName) + } + + func testFindEntityForHost() { + let exampleData = Self.exampleTDS.data(using: .utf8)! + let embeddedDataProvider = MockEmbeddedDataProvider(data: exampleData, + etag: "embedded") + + let trackerDataManager = TrackerDataManager(etag: nil, + data: nil, + embeddedDataProvider: embeddedDataProvider) + + let entity = trackerDataManager.embeddedData.tds.findEntity(forHost: "www.notreal.io") + XCTAssertNotNil(entity) + XCTAssertEqual("Not Real", entity?.displayName) + } + + // swiftlint:disable function_body_length + func testWhenDownloadedDataAvailableThenReloadUsesIt() { + + let exampleData = Self.exampleTDS.data(using: .utf8)! + let embeddedDataProvider = MockEmbeddedDataProvider(data: exampleData, + etag: "embedded") + + let trackerDataManager = TrackerDataManager(etag: nil, + data: nil, + embeddedDataProvider: embeddedDataProvider) + + XCTAssertEqual(trackerDataManager.embeddedData.etag, "embedded") + XCTAssertEqual(trackerDataManager.reload(etag: "new etag", data: exampleData), + TrackerDataManager.ReloadResult.downloaded) + + XCTAssertEqual(trackerDataManager.fetchedData?.etag, "new etag") + XCTAssertNil(trackerDataManager.fetchedData?.tds.findEntity(byName: "Google LLC")) + XCTAssertNotNil(trackerDataManager.fetchedData?.tds.findEntity(byName: "Not Real")) + + } + // swiftlint:enable function_body_length +} diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/TrackerResolverTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerResolverTests.swift new file mode 100644 index 000000000..48fe00ac6 --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerResolverTests.swift @@ -0,0 +1,231 @@ +// +// TrackerResolverTests.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import TrackerRadarKit +@testable import BrowserServicesKit + +class TrackerResolverTests: XCTestCase { + + func testWhenOptionsAreEmptyThenNothingMatches() { + + let rule = KnownTracker.Rule.Matching(domains: [], types: []) + + let urlOne = URL(string: "https://www.one.com")! + + XCTAssertFalse(TrackerResolver.isMatching(rule, + host: urlOne.host!, + resourceType: "image")) + } + + func testWhenDomainsAreRequiredThenTypesDoNotMatter() { + + let rule = KnownTracker.Rule.Matching(domains: ["one.com", "two.com"], types: nil) + + let urlOne = URL(string: "https://www.one.com")! + let urlTwo = URL(string: "https://two.com")! + let urlThree = URL(string: "https://www.three.com")! + + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlOne.host!, + resourceType: "image")) + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlTwo.host!, + resourceType: "image")) + XCTAssertFalse(TrackerResolver.isMatching(rule, + host: urlThree.host!, + resourceType: "image")) + } + + func testWhenTypesAreRequiredThenDomainsDoNotMatter() { + + let rule = KnownTracker.Rule.Matching(domains: [], types: ["image", "script"]) + + let urlOne = URL(string: "https://www.one.com")! + let urlTwo = URL(string: "https://two.com")! + let urlThree = URL(string: "https://www.three.com")! + + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlOne.host!, + resourceType: "image")) + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlTwo.host!, + resourceType: "script")) + XCTAssertFalse(TrackerResolver.isMatching(rule, + host: urlThree.host!, + resourceType: "link")) + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlThree.host!, + resourceType: "image")) + } + + func testWhenTypesAndDomainsAreRequiredThenItIsAnAndRequirement() { + + let rule = KnownTracker.Rule.Matching(domains: ["one.com", "two.com"], types: ["image", "script"]) + + let urlOne = URL(string: "https://www.one.com")! + let urlTwo = URL(string: "https://two.com")! + let urlThree = URL(string: "https://www.three.com")! + + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlOne.host!, + resourceType: "image")) + XCTAssertFalse(TrackerResolver.isMatching(rule, + host: urlOne.host!, + resourceType: "link")) + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlOne.host!, + resourceType: "script")) + + XCTAssertTrue(TrackerResolver.isMatching(rule, + host: urlTwo.host!, + resourceType: "script")) + XCTAssertFalse(TrackerResolver.isMatching(rule, + host: urlTwo.host!, + resourceType: "link")) + + XCTAssertFalse(TrackerResolver.isMatching(rule, + host: urlThree.host!, + resourceType: "link")) + XCTAssertFalse(TrackerResolver.isMatching(rule, + host: urlThree.host!, + resourceType: "image")) + } + + func testWhenTrackerIsDetectedThenItIsReported() { + + let tracker = KnownTracker(domain: "tracker.com", + defaultAction: .block, + owner: KnownTracker.Owner(name: "Tracker Inc", + displayName: "Tracker Inc company"), + prevalence: 0.1, + subdomains: nil, + categories: nil, + rules: nil) + + let tds = TrackerData(trackers: ["tracker.com" : tracker], + entities: ["Tracker Inc": Entity(displayName: "Trackr Inc company", + domains: ["tracker.com"], + prevalence: 0.1)], + domains: ["tracker.com": "Tracker Inc"], + cnames: [:]) + + let resolver = TrackerResolver(tds: tds, unprotectedSites: [], tempList: []) + + let result = resolver.trackerFromUrl("https://tracker.com/img/1.png", pageUrlString: "example.com", resourceType: "image", potentiallyBlocked: true) + + XCTAssertNotNil(result) + XCTAssert(result?.blocked ?? false) + XCTAssertEqual(result?.knownTracker, tracker) + } + + func testWhenTrackerIsOnAssociatedPageThenItIsNotReported() { + + let tracker = KnownTracker(domain: "tracker.com", + defaultAction: .block, + owner: KnownTracker.Owner(name: "Tracker Inc", + displayName: "Tracker Inc company"), + prevalence: 0.1, + subdomains: nil, + categories: nil, + rules: nil) + + let tds = TrackerData(trackers: ["tracker.com" : tracker], + entities: ["Tracker Inc": Entity(displayName: "Tracker Inc company", + domains: ["tracker.com", "example.com"], + prevalence: 0.1)], + domains: ["tracker.com": "Tracker Inc", + "example.com": "Tracker Inc"], + cnames: [:]) + + let resolver = TrackerResolver(tds: tds, unprotectedSites: [], tempList: []) + + let result = resolver.trackerFromUrl("https://tracker.com/img/1.png", pageUrlString: "https://example.com", resourceType: "image", potentiallyBlocked: true) + + XCTAssertNil(result) + } + + func testWhenTrackerIsACnameThenItIsReportedAsSuch() { + + let tracker = KnownTracker(domain: "tracker.com", + defaultAction: .block, + owner: KnownTracker.Owner(name: "Tracker Inc", + displayName: "Tracker Inc company"), + prevalence: 0.1, + subdomains: nil, + categories: nil, + rules: nil) + + let tds = TrackerData(trackers: ["tracker.com" : tracker], + entities: ["Tracker Inc": Entity(displayName: "Tracker Inc company", + domains: ["tracker.com"], + prevalence: 0.1)], + domains: ["tracker.com": "Tracker Inc"], + cnames: ["cnamed.com": "tracker.com"]) + + let resolver = TrackerResolver(tds: tds, unprotectedSites: [], tempList: []) + + let result = resolver.trackerFromUrl("https://cnamed.com/img/1.png", pageUrlString: "https://example.com", resourceType: "image", potentiallyBlocked: true) + + XCTAssertNotNil(result) + XCTAssert(result?.blocked ?? false) + XCTAssertEqual(result?.knownTracker, tracker) + } + + func testWhenTrackerIsACnameForAnotherTrackerThenOriginalOneIsReturned() { + + let tracker = KnownTracker(domain: "tracker.com", + defaultAction: .block, + owner: KnownTracker.Owner(name: "Tracker Inc", + displayName: "Tracker Inc company"), + prevalence: 0.1, + subdomains: nil, + categories: nil, + rules: nil) + + let another = KnownTracker(domain: "another.com", + defaultAction: .block, + owner: KnownTracker.Owner(name: "Another Inc", + displayName: "Another Inc company"), + prevalence: 0.1, + subdomains: nil, + categories: nil, + rules: nil) + + let tds = TrackerData(trackers: ["tracker.com" : tracker, + "another.com" : another], + entities: ["Tracker Inc": Entity(displayName: "Tracker Inc company", + domains: ["tracker.com"], + prevalence: 0.1), + "Another Inc": Entity(displayName: "Another Inc company", + domains: ["another.com"], + prevalence: 0.1)], + domains: ["tracker.com": "Tracker Inc", + "another.com": "Another Inc."], + cnames: ["sub.another.com": "tracker.com"]) + + let resolver = TrackerResolver(tds: tds, unprotectedSites: [], tempList: []) + + let result = resolver.trackerFromUrl("https://sub.another.com/img/1.png", pageUrlString: "https://example.com", resourceType: "image", potentiallyBlocked: true) + + XCTAssertNotNil(result) + XCTAssert(result?.blocked ?? false) + XCTAssertEqual(result?.knownTracker, another) + } +} diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift b/Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift new file mode 100644 index 000000000..acc7b59e5 --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift @@ -0,0 +1,200 @@ +// +// WebViewTestHelper.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import WebKit +import XCTest +import BrowserServicesKit +import TrackerRadarKit + +final class MockNavigationDelegate: NSObject, WKNavigationDelegate { + + var onDidFinishNavigation: (() -> Void)? + + func webView(_ webView: WKWebView, didFail navigation: WKNavigation!, withError error: Error) { + XCTFail("Could to navigate to test site") + } + + func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) { + onDidFinishNavigation?() + } +} + +final class MockRulesUserScriptDelegate: NSObject, ContentBlockerRulesUserScriptDelegate { + + var shouldProcessTrackers = true + var onTrackerDetected: ((DetectedTracker) -> Void)? + var detectedTrackers = Set() + + func reset() { + detectedTrackers.removeAll() + } + + func contentBlockerRulesUserScriptShouldProcessTrackers(_ script: ContentBlockerRulesUserScript) -> Bool { + return shouldProcessTrackers + } + + func contentBlockerRulesUserScriptShouldProcessCTLTrackers(_ script: ContentBlockerRulesUserScript) -> Bool { + return true + } + + func contentBlockerRulesUserScript(_ script: ContentBlockerRulesUserScript, + detectedTracker tracker: DetectedTracker) { + detectedTrackers.insert(tracker) + onTrackerDetected?(tracker) + } +} + +final class MockSurrogatesUserScriptDelegate: NSObject, SurrogatesUserScriptDelegate { + + var shouldProcessTrackers = true + + var onSurrogateDetected: ((DetectedTracker, String) -> Void)? + var detectedSurrogates = Set() + + func reset() { + detectedSurrogates.removeAll() + } + + func surrogatesUserScriptShouldProcessTrackers(_ script: SurrogatesUserScript) -> Bool { + return shouldProcessTrackers + } + + func surrogatesUserScript(_ script: SurrogatesUserScript, + detectedTracker tracker: DetectedTracker, + withSurrogate host: String) { + detectedSurrogates.insert(tracker) + onSurrogateDetected?(tracker, host) + } +} + +final class MockDomainsProtectionStore: DomainsProtectionStore { + var unprotectedDomains = Set() + + func disableProtection(forDomain domain: String) { + unprotectedDomains.insert(domain) + } + + func enableProtection(forDomain domain: String) { + unprotectedDomains.remove(domain) + } +} + +final class TestSchemeContentBlockerUserScriptConfig: ContentBlockerUserScriptConfig { + + public let privacyConfiguration: PrivacyConfiguration + public let trackerData: TrackerData? + public let ctlTrackerData: TrackerData? + + public private(set) var source: String + + public init(privacyConfiguration: PrivacyConfiguration, + trackerData: TrackerData?, + ctlTrackerData: TrackerData?) { + self.privacyConfiguration = privacyConfiguration + self.trackerData = trackerData + self.ctlTrackerData = ctlTrackerData + + // UserScripts contain TrackerAllowlist rules in form of regular expressions - we need to ensure test scheme is matched instead of http/https + let orginalSource = ContentBlockerRulesUserScript.generateSource(privacyConfiguration: privacyConfiguration) + source = orginalSource.replacingOccurrences(of: "http", with: "test") + } +} + +public class TestSchemeSurrogatesUserScriptConfig: SurrogatesUserScriptConfig { + + public let privacyConfig: PrivacyConfiguration + public let surrogates: String + public let trackerData: TrackerData? + public let encodedSurrogateTrackerData: String? + + public let source: String + + public init(privacyConfig: PrivacyConfiguration, + surrogates: String, + trackerData: TrackerData?, + encodedSurrogateTrackerData: String?, + isDebugBuild: Bool) { + + self.privacyConfig = privacyConfig + self.surrogates = surrogates + self.trackerData = trackerData + self.encodedSurrogateTrackerData = encodedSurrogateTrackerData + + // UserScripts contain TrackerAllowlist rules in form of regular expressions - we need to ensure test scheme is matched instead of http/https + let orginalSource = SurrogatesUserScript.generateSource(privacyConfiguration: privacyConfig, + surrogates: surrogates, + encodedSurrogateTrackerData: encodedSurrogateTrackerData, + isDebugBuild: isDebugBuild) + + source = orginalSource.replacingOccurrences(of: "http", with: "test") + } +} + +final class WebKitTestHelper { + + static func preparePrivacyConfig(locallyUnprotected: [String], + tempUnprotected: [String], + trackerAllowlist: [String: [PrivacyConfigurationData.TrackerAllowlist.Entry]], + contentBlockingEnabled: Bool, + exceptions: [String], + httpsUpgradesEnabled: Bool = false) -> PrivacyConfiguration { + let contentBlockingExceptions = exceptions.map { PrivacyConfigurationData.ExceptionEntry(domain: $0, reason: nil) } + let contentBlockingStatus = contentBlockingEnabled ? "enabled" : "disabled" + let httpsStatus = httpsUpgradesEnabled ? "enabled" : "disabled" + let features = [PrivacyFeature.contentBlocking.rawValue: PrivacyConfigurationData.PrivacyFeature(state: contentBlockingStatus, + exceptions: contentBlockingExceptions), + PrivacyFeature.httpsUpgrade.rawValue: PrivacyConfigurationData.PrivacyFeature(state: httpsStatus, exceptions: [])] + let unprotectedTemporary = tempUnprotected.map { PrivacyConfigurationData.ExceptionEntry(domain: $0, reason: nil) } + let privacyData = PrivacyConfigurationData(features: features, + unprotectedTemporary: unprotectedTemporary, + trackerAllowlist: trackerAllowlist) + + let localProtection = MockDomainsProtectionStore() + localProtection.unprotectedDomains = Set(locallyUnprotected) + + return AppPrivacyConfiguration(data: privacyData, + identifier: "", + localProtection: localProtection) + } + + static func prepareContentBlockingRules(trackerData: TrackerData, + exceptions: [String], + tempUnprotected: [String], + trackerExceptions: [TrackerException], + completion: @escaping (WKContentRuleList?) -> Void) { + + let rules = ContentBlockerRulesBuilder(trackerData: trackerData).buildRules(withExceptions: exceptions, + andTemporaryUnprotectedDomains: tempUnprotected, + andTrackerAllowlist: trackerExceptions) + + let data = (try? JSONEncoder().encode(rules))! + var ruleList = String(data: data, encoding: .utf8)! + + // Replace https scheme regexp with test + ruleList = ruleList.replacingOccurrences(of: "https", with: "test", options: [], range: nil) + + WKContentRuleListStore.default().compileContentRuleList(forIdentifier: "test", encodedContentRuleList: ruleList) { list, _ in + + DispatchQueue.main.async { + completion(list) + } + } + } +} diff --git a/Tests/BrowserServicesKitTests/Email/EmailManagerTests.swift b/Tests/BrowserServicesKitTests/Email/EmailManagerTests.swift index 08723e9ac..3915281b3 100644 --- a/Tests/BrowserServicesKitTests/Email/EmailManagerTests.swift +++ b/Tests/BrowserServicesKitTests/Email/EmailManagerTests.swift @@ -582,8 +582,8 @@ class MockEmailManagerRequestDelegate: EmailManagerRequestDelegate { func emailManager(_ emailManager: EmailManager, requested url: URL, method: String, - headers: [String : String], - parameters: [String : String]?, + headers: [String: String], + parameters: [String: String]?, httpBody: Data?, timeoutInterval: TimeInterval, completion: @escaping (Data?, Error?) -> Void) { diff --git a/Tests/BrowserServicesKitTests/PrivacyConfig/AppPrivacyConfigurationTests.swift b/Tests/BrowserServicesKitTests/PrivacyConfig/AppPrivacyConfigurationTests.swift new file mode 100644 index 000000000..01088fd58 --- /dev/null +++ b/Tests/BrowserServicesKitTests/PrivacyConfig/AppPrivacyConfigurationTests.swift @@ -0,0 +1,264 @@ +// +// AppPrivacyConfigurationTests.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import BrowserServicesKit + +class MockEmbeddedDataProvider: EmbeddedDataProvider { + var embeddedDataEtag: String + + var embeddedData: Data + + init(data: Data, etag: String) { + embeddedData = data + embeddedDataEtag = etag + } +} + +class AppPrivacyConfigurationTests: XCTestCase { + + let embeddedConfig = + """ + { + "features": {}, + "unprotectedTemporary": [ + { "domain": "domain1.com" }, + { "domain": "domain2.com" }, + { "domain": "domain3.com" }, + ] + } + """.data(using: .utf8)! + let embeddedConfigETag = "embedded" + + let downloadedConfig = + """ + { + "features": {}, + "unprotectedTemporary": [ + { "domain": "domain1.com" }, + { "domain": "domain5.com" }, + { "domain": "domain6.com" }, + ] + } + """.data(using: .utf8)! + let downloadedConfigETag = "downloaded" + + let corruptedConfig = + """ + { + "features": {}, + "unprotectedTemporary": [ + } + """.data(using: .utf8)! + let corruptedConfigETag = "corrupted" + + func testWhenDownloadedDataIsMissing_ThenEmbeddedIsUsed() { + + let mockEmbeddedData = MockEmbeddedDataProvider(data: embeddedConfig, etag: embeddedConfigETag) + + let manager = PrivacyConfigurationManager(fetchedETag: nil, + fetchedData: nil, + embeddedDataProvider: mockEmbeddedData, + localProtection: MockDomainsProtectionStore()) + + XCTAssertEqual(manager.embeddedConfigData.etag, mockEmbeddedData.embeddedDataEtag) + XCTAssertEqual(manager.reload(etag: nil, data: nil), PrivacyConfigurationManager.ReloadResult.embedded) + + XCTAssertNil(manager.fetchedConfigData) + + let config = manager.privacyConfig + XCTAssertFalse(config.isTempUnprotected(domain: "main1.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "notdomain1.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain1.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain2.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "domain5.com")) + + XCTAssertTrue(config.isTempUnprotected(domain: "www.domain1.com")) + } + + func testWhenDataIsPresent_ThenItIsUsed() { + + let mockEmbeddedData = MockEmbeddedDataProvider(data: embeddedConfig, etag: embeddedConfigETag) + + let manager = PrivacyConfigurationManager(fetchedETag: downloadedConfigETag, + fetchedData: downloadedConfig, + embeddedDataProvider: mockEmbeddedData, + localProtection: MockDomainsProtectionStore()) + + XCTAssertEqual(manager.embeddedConfigData.etag, mockEmbeddedData.embeddedDataEtag) + XCTAssertEqual(manager.fetchedConfigData?.etag, downloadedConfigETag) + + let config = manager.privacyConfig + XCTAssertFalse(config.isTempUnprotected(domain: "main1.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "notdomain1.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain1.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "domain2.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain5.com")) + + XCTAssertTrue(config.isTempUnprotected(domain: "www.domain1.com")) + } + + func testWhenDownloadedDataIsReloaded_ThenItIsUsed() { + + let mockEmbeddedData = MockEmbeddedDataProvider(data: embeddedConfig, etag: embeddedConfigETag) + + let manager = PrivacyConfigurationManager(fetchedETag: nil, + fetchedData: nil, + embeddedDataProvider: mockEmbeddedData, + localProtection: MockDomainsProtectionStore()) + + XCTAssertEqual(manager.embeddedConfigData.etag, mockEmbeddedData.embeddedDataEtag) + XCTAssertNil(manager.fetchedConfigData) + + XCTAssertEqual(manager.reload(etag: downloadedConfigETag, data: downloadedConfig), .downloaded) + + let config = manager.privacyConfig + XCTAssertFalse(config.isTempUnprotected(domain: "main1.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "notdomain1.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain1.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "domain2.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain5.com")) + + XCTAssertTrue(config.isTempUnprotected(domain: "www.domain1.com")) + } + + func testWhenPresentDataIsCorrupted_ThenEmbeddedIsUsed() { + + let mockEmbeddedData = MockEmbeddedDataProvider(data: embeddedConfig, etag: embeddedConfigETag) + + let manager = PrivacyConfigurationManager(fetchedETag: corruptedConfigETag, + fetchedData: corruptedConfig, + embeddedDataProvider: mockEmbeddedData, + localProtection: MockDomainsProtectionStore()) + + XCTAssertEqual(manager.embeddedConfigData.etag, mockEmbeddedData.embeddedDataEtag) + XCTAssertNil(manager.fetchedConfigData) + + // Should use embedded + var config = manager.privacyConfig + XCTAssertTrue(config.isTempUnprotected(domain: "domain1.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain2.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "domain5.com")) + + // Attempt fix + XCTAssertEqual(manager.reload(etag: downloadedConfigETag, data: downloadedConfig), .downloaded) + + config = manager.privacyConfig + XCTAssertTrue(config.isTempUnprotected(domain: "domain1.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "domain2.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain5.com")) + } + + func testWhenReloadedDataIsCorrupted_ThenEmbeddedIsUsed() { + + let mockEmbeddedData = MockEmbeddedDataProvider(data: embeddedConfig, etag: embeddedConfigETag) + + let manager = PrivacyConfigurationManager(fetchedETag: nil, + fetchedData: nil, + embeddedDataProvider: mockEmbeddedData, + localProtection: MockDomainsProtectionStore()) + + XCTAssertEqual(manager.embeddedConfigData.etag, mockEmbeddedData.embeddedDataEtag) + XCTAssertNil(manager.fetchedConfigData) + + // Should use embedded + var config = manager.privacyConfig + XCTAssertTrue(config.isTempUnprotected(domain: "domain1.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain2.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "domain5.com")) + + // Attempt fix + XCTAssertEqual(manager.reload(etag: corruptedConfigETag, data: corruptedConfig), .embeddedFallback) + + config = manager.privacyConfig + XCTAssertTrue(config.isTempUnprotected(domain: "domain1.com")) + XCTAssertTrue(config.isTempUnprotected(domain: "domain2.com")) + XCTAssertFalse(config.isTempUnprotected(domain: "domain5.com")) + } + + func testWhenCheckingUnprotectedSites_ThenProtectionStoreIsUsed() { + + let mockEmbeddedData = MockEmbeddedDataProvider(data: embeddedConfig, etag: embeddedConfigETag) + + let mockProtectionStore = MockDomainsProtectionStore() + mockProtectionStore.disableProtection(forDomain: "enabled.com") + + let manager = PrivacyConfigurationManager(fetchedETag: corruptedConfigETag, + fetchedData: corruptedConfig, + embeddedDataProvider: mockEmbeddedData, + localProtection: mockProtectionStore) + + XCTAssertEqual(manager.embeddedConfigData.etag, mockEmbeddedData.embeddedDataEtag) + XCTAssertNil(manager.fetchedConfigData) + + let config = manager.privacyConfig + + XCTAssertTrue(config.isUserUnprotected(domain: "enabled.com")) + + config.userDisabledProtection(forDomain: "enabled2.com") + XCTAssertTrue(config.isUserUnprotected(domain: "enabled2.com")) + } + + let exampleConfig = + """ + { + "features": { + "gpc": { + "state": "enabled", + "exceptions": [ + { + "domain": "example.com", + "reason": "site breakage" + } + ] + } + }, + "unprotectedTemporary": [ + { + "domain": "unp.com", + "reason": "site breakage" + } + ] + } + """.data(using: .utf8)! + + func testWhenCheckingFeatureState_ThenValidStateIsReturned() { + + let mockEmbeddedData = MockEmbeddedDataProvider(data: exampleConfig, etag: "test") + + let mockProtectionStore = MockDomainsProtectionStore() + mockProtectionStore.disableProtection(forDomain: "disabled.com") + + let manager = PrivacyConfigurationManager(fetchedETag: nil, + fetchedData: nil, + embeddedDataProvider: mockEmbeddedData, + localProtection: mockProtectionStore) + + XCTAssertEqual(manager.embeddedConfigData.etag, mockEmbeddedData.embeddedDataEtag) + XCTAssertNil(manager.fetchedConfigData) + + let config = manager.privacyConfig + + XCTAssertTrue(config.isFeature(.gpc, enabledForDomain: nil)) + XCTAssertTrue(config.isFeature(.gpc, enabledForDomain: "test.com")) + XCTAssertFalse(config.isFeature(.gpc, enabledForDomain: "disabled.com")) + XCTAssertFalse(config.isFeature(.gpc, enabledForDomain: "example.com")) + XCTAssertFalse(config.isFeature(.gpc, enabledForDomain: "unp.com")) + } +} diff --git a/Tests/BrowserServicesKitTests/PrivacyConfig/PrivacyConfigurationDataTests.swift b/Tests/BrowserServicesKitTests/PrivacyConfig/PrivacyConfigurationDataTests.swift new file mode 100644 index 000000000..6ea54df9a --- /dev/null +++ b/Tests/BrowserServicesKitTests/PrivacyConfig/PrivacyConfigurationDataTests.swift @@ -0,0 +1,90 @@ +// +// PrivacyConfigurationDataTests.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import CommonCrypto +@testable import BrowserServicesKit + +class PrivacyConfigurationDataTests: XCTestCase { + + private var data = JsonTestDataLoader() + + func testJSONParsing() { + let jsonData = data.fromJsonFile("privacy-config-example.json") + let json = try? JSONSerialization.jsonObject(with: jsonData, options: []) as? [String: Any] + + let configData = PrivacyConfigurationData(json: json!) + + XCTAssertEqual(configData.unprotectedTemporary.count, 1) + XCTAssertEqual(configData.unprotectedTemporary.first?.domain, "example.com") + + let gpcFeature = configData.features["contentBlocking"] + XCTAssertNotNil(gpcFeature) + XCTAssertEqual(gpcFeature?.state, "enabled") + XCTAssertEqual(gpcFeature?.exceptions.first?.domain, "example.com") + + let exampleFeature = configData.features["exampleFeature"] + XCTAssertEqual(exampleFeature?.state, "enabled") + XCTAssertEqual((exampleFeature?.settings["dictValue"] as? [String: String])?["key"], "value") + XCTAssertEqual((exampleFeature?.settings["arrayValue"] as? [String])?.first, "value") + XCTAssertEqual((exampleFeature?.settings["stringValue"] as? String), "value") + XCTAssertEqual((exampleFeature?.settings["numericalValue"] as? Int), 1) + + let allowlist = configData.trackerAllowlist + XCTAssertEqual(allowlist.state, "enabled") + let rulesMap = allowlist.entries.reduce(into: [String: [String]]()) { partialResult, entry in + for e in entry.value { + partialResult[e.rule] = e.domains + } + } + XCTAssertEqual(rulesMap["example.com/tracker.js"], ["test.com"]) + XCTAssertEqual(rulesMap["example2.com/path/"], [""]) + XCTAssertEqual(rulesMap["example2.com/resource.json"], [""]) + } + + func testJSONWithoutAllowlistParsing() { + let jsonData = data.fromJsonFile("privacy-config-example.json") + var json = try? JSONSerialization.jsonObject(with: jsonData, options: []) as? [String: Any] + var features = json?["features"] as? [String: Any] + features?.removeValue(forKey: "trackerAllowlist") + json?["features"] = features + + let configData = PrivacyConfigurationData(json: json!) + + XCTAssertEqual(configData.unprotectedTemporary.count, 1) + XCTAssertEqual(configData.unprotectedTemporary.first?.domain, "example.com") + + let gpcFeature = configData.features["contentBlocking"] + XCTAssertNotNil(gpcFeature) + XCTAssertEqual(gpcFeature?.state, "enabled") + XCTAssertEqual(gpcFeature?.exceptions.first?.domain, "example.com") + + let exampleFeature = configData.features["exampleFeature"] + XCTAssertEqual(exampleFeature?.state, "enabled") + XCTAssertEqual((exampleFeature?.settings["dictValue"] as? [String: String])?["key"], "value") + XCTAssertEqual((exampleFeature?.settings["arrayValue"] as? [String])?.first, "value") + XCTAssertEqual((exampleFeature?.settings["stringValue"] as? String), "value") + XCTAssertEqual((exampleFeature?.settings["numericalValue"] as? Int), 1) + + let allowlist = configData.trackerAllowlist + XCTAssertEqual(allowlist.state, "disabled") + XCTAssertEqual(allowlist.entries.count, 0) + } + +} diff --git a/Tests/BrowserServicesKitTests/Resources/domain_matching_tests.json b/Tests/BrowserServicesKitTests/Resources/domain_matching_tests.json new file mode 100644 index 000000000..488f5fbb4 --- /dev/null +++ b/Tests/BrowserServicesKitTests/Resources/domain_matching_tests.json @@ -0,0 +1,230 @@ +{ + "domainTests": { + "name": "URL-matching", + "desc": "interactions between root domain and resource URLs from tracker lists", + "tests": [ + { + "name": "same party tracker", + "siteURL": "https://bad.third-party.site/", + "requestURL": "https://bad.third-party.site/", + "requestType": "script", + "expectAction": "ignore", + "exceptPlatforms": [] + }, + { + "name": "bad loads same root domain resources", + "siteURL": "https://bad.third-party.site/", + "requestURL": "https://third-party.site/", + "requestType": "script", + "expectAction": null, + "exceptPlatforms": [] + }, + { + "name": "root domain matches itself, not a subdomain", + "siteURL": "https://third-party.site/", + "requestURL": "https://third-party.site/stuff", + "requestType": "script", + "expectAction": null, + "exceptPlatforms": [] + }, + { + "name": "same party ignore", + "siteURL": "https://ignore.test/", + "requestURL": "https://ignore.test/", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "tracker loads ignore", + "siteURL": "https://bad.third-party.site/", + "requestURL": "https://ignore.test/", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "ignore image exception loads tracker image", + "siteURL": "https://ignore.test/", + "requestURL": "https://bad.third-party.site/", + "requestType": "image", + "expectAction": "ignore" + }, + { + "name": "ignore image exception doesn't load tracker script", + "siteURL": "https://ignore.test/", + "requestURL": "https://bad.third-party.site/", + "requestType": "script", + "expectAction": "block" + }, + { + "name": "notignore doesn't load tracker", + "siteURL": "https://notignore.test/", + "requestURL": "https://bad.third-party.site/", + "requestType": "script", + "expectAction": "block" + }, + { + "name": "random blocks tracker", + "siteURL": "https://randomsite123.com/", + "requestURL": "https://bad.third-party.site/", + "requestType": "script", + "expectAction": "block" + }, + { + "name": "random blocks cname tracker", + "siteURL": "https://randomsite123.com/", + "requestURL": "https://bad.cnames.test/something", + "requestType": "script", + "expectAction": "block" + }, + { + "name": "random allows sub.cname tracker", + "siteURL": "https://randomsite123.com/", + "requestURL": "https://also.bad.cnames.test/something", + "requestType": "script", + "expectAction": null, + "exceptPlatforms": [ + "ios-browser" + ] + }, + { + "name": "random allows notbad cname tracker", + "siteURL": "https://randomsite123.com/", + "requestURL": "https://notbad.cnames.test/something", + "requestType": "script", + "expectAction": null + }, + { + "name": "sub.ignore loads tracker", + "siteURL": "https://sub.ignore.test/", + "requestURL": "https://bad.third-party.site/", + "requestType": "image", + "expectAction": "ignore" + }, + { + "name": "tracker loads ignore resource type", + "siteURL": "https://bad.third-party.site/", + "requestURL": "https://ignore.test/", + "requestType": "image", + "expectAction": "ignore" + }, + { + "name": "tracker blocks non-ignore resource type", + "siteURL": "https://bad.third-party.site/", + "requestURL": "https://ignore.test/", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "tracker loads random", + "siteURL": "https://bad.third-party.site/", + "requestURL": "https://random.test/", + "requestType": "script", + "expectAction": null + }, + { + "name": "random does not load tracker", + "siteURL": "https://random.test/", + "requestURL": "https://bad.third-party.site/", + "requestType": "script", + "expectAction": "block" + }, + { + "name": "random loads itself", + "siteURL": "https://random.test/", + "requestURL": "https://random.test/", + "requestType": "script", + "expectAction": null + }, + { + "name": "random loads ignore path on tracker", + "siteURL": "https://random.test/", + "requestURL": "https://bad.third-party.site/ignore", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "random loads broken.site", + "siteURL": "https://random.test/", + "requestURL": "https://broken.third-party.site/", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "random loads randomsub.third-party.site", + "siteURL": "https://random.test/", + "requestURL": "https://randomsub.third-party.site/", + "requestType": "script", + "expectAction": null + }, + { + "name": "random doesn't load randomsub.tracker", + "siteURL": "https://random.test/", + "requestURL": "https://randomsub.tracker.test/", + "requestType": "script", + "expectAction": "block" + }, + { + "name": "tracker loads permitted for same entity", + "siteURL": "https://third-party.site/", + "requestURL": "https://tracker.test/", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "block tracker path in ignore domain", + "siteURL": "https://random.test/", + "requestURL": "https://ignore.test/tracker?abc=2", + "requestType": "script", + "expectAction": "block" + } + ] + }, + "surrogateTests": { + "name": "surrogate-tests", + "desc": "interactions between root domain and resource URLs for surrogates", + "tests": [ + { + "name": "handle surrogates", + "siteURL": "https://random.test/", + "requestURL": "https://surrogates.test/tracker?abc=2", + "requestType": "script", + "expectAction": "redirect", + "expectRedirect": "data:application/javascript;base64,KGZ1bmN0aW9uKCkge3ZhciB0cmFja2VyPXRydWV9KSgpOw==" + }, + { + "name": "skip surrogates for same entity", + "siteURL": "https://tracker.test/", + "requestURL": "https://surrogates.test/tracker?abc=2", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "surrogates match on substrings", + "siteURL": "https://random.test/", + "requestURL": "https://surrogates.test/trackerNOT?abc=2", + "requestType": "script", + "expectAction": "redirect", + "expectRedirect": "data:application/javascript;base64,KGZ1bmN0aW9uKCkge3ZhciB0cmFja2VyPXRydWV9KSgpOw==" + }, + { + "name": "require exact match for surrogates", + "siteURL": "https://random.test/", + "requestURL": "https://surrogates.test/NOTtracker?abc=2", + "requestType": "script", + "expectAction": "ignore" + }, + { + "name": "block on undefined surrogates", + "siteURL": "https://random.test/", + "requestURL": "https://surrogates.test/anothertracker?abc=2", + "requestType": "script", + "expectAction": "block" + } + ] + }, + "safeTests": { + "name": "safe-matching", + "desc": "tests for safelist interactions", + "tests": [] + } +} diff --git a/Tests/BrowserServicesKitTests/Resources/privacy-config-example.json b/Tests/BrowserServicesKitTests/Resources/privacy-config-example.json new file mode 100644 index 000000000..4da940232 --- /dev/null +++ b/Tests/BrowserServicesKitTests/Resources/privacy-config-example.json @@ -0,0 +1,162 @@ +{ + "version": "2021.6.7", + "readme": "https://github.com/duckduckgo/privacy-configuration", + "features": { + "contentBlocking": { + "state": "enabled", + "exceptions": [ + { + "domain": "example.com", + "reason": "Adblocker wall" + } + ] + }, + "trackingCookies3p": { + "state": "enabled", + "exceptions": [ + { + "domain": "example.com", + "reason": "site breakage" + } + ], + "settings": { + "excludedCookieDomains": [ + { + "domain": "example.com", + "reason": "Site breakage" + } + ] + } + }, + "trackingCookies1p": { + "state": "enabled", + "settings": { + "firstPartyTrackerCookiePolicy": { + "threshold": 86400, + "maxAge": 86400 + } + }, + "exceptions": [] + }, + "clickToPlay": { + "state": "enabled", + "exceptions": [] + }, + "fingerprintingCanvas": { + "state": "enabled", + "exceptions": [ + { + "domain": "example.com", + "reason": "site breakage" + } + ] + }, + "fingerprintingAudio": { + "state": "disabled", + "exceptions": [ + { + "domain": "example.com" + } + ] + }, + "fingerprintingTemporaryStorage": { + "state": "enabled", + "exceptions": [] + }, + "referrer": { + "state": "enabled", + "exceptions": [] + }, + "fingerprintingBattery": { + "state": "enabled", + "exceptions": [] + }, + "fingerprintingScreenSize": { + "state": "enabled", + "exceptions": [] + }, + "fingerprintingHardware": { + "state": "enabled", + "exceptions": [ + { + "domain": "example.com" + } + ] + }, + "floc": { + "state": "enabled", + "exceptions": [] + }, + "gpc": { + "state": "enabled", + "exceptions": [] + }, + "userAgentRotation": { + "state": "disabled", + "settings": { + "agentExcludePatterns": [ + { + "agent": "Brave Chrome", + "reason": "Uncommon UA" + } + ] + }, + "exceptions": [ + { + "domain": "example.com", + "reason": "Two factor auth that verifies device pathes using user agent" + } + ] + }, + "trackerAllowlist": { + "state": "enabled", + "settings": { + "allowlistedTrackers": { + "example.com": { + "rules" : [ + { + "rule": "example.com/tracker.js", + "domains": ["test.com"], + "reason": "broken" + } + ] + }, + "example2.com": { + "rules" : [ + { + "rule": "example2.com/path/", + "domains": [""], + "reason": "broken" + }, + { + "rule": "example2.com/resource.json", + "domains": [""], + "reason": "broken" + } + ] + } + } + } + }, + "exampleFeature": { + "state": "enabled", + "exceptions": [], + "settings": { + "dictValue": { + "key": "value" + }, + "arrayValue": [ + "value" + ], + "stringValue": "value", + "numericalValue": 1 + }, + } + }, + "unprotectedTemporary": [ + { + "domain": "example.com", + "reason": "site breakage" + } + ] +} diff --git a/Tests/BrowserServicesKitTests/Resources/surrogates.js b/Tests/BrowserServicesKitTests/Resources/surrogates.js new file mode 100644 index 000000000..d87556cb1 --- /dev/null +++ b/Tests/BrowserServicesKitTests/Resources/surrogates.js @@ -0,0 +1,3 @@ +module.exports = { + surrogates: 'surrogates.test/tracker application/javascript\n(function() {var tracker=true})();' +} diff --git a/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_matching_tests.json b/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_matching_tests.json new file mode 100644 index 000000000..40effe7c5 --- /dev/null +++ b/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_matching_tests.json @@ -0,0 +1,86 @@ +[ + { + "description": "should allow single resource on single site", + "site": "https://testsite.com", + "request": "https://allowlist-tracker-1.com/videos.js", + "isAllowlisted": true + }, + { + "description": "should match on all subdomains of an allowlisted site", + "site": "https://a.b.c.testsite.com", + "request": "https://allowlist-tracker-1.com/videos.js", + "isAllowlisted": true + }, + { + "description": "should remove port from the tracker request", + "site": "https://a.b.c.testsite.com", + "request": "https://allowlist-tracker-1.com:5000/videos.js", + "isAllowlisted": true + }, + { + "description": "should remove parameter string from the tracker request", + "site": "https://a.b.c.testsite.com", + "request": "https://allowlist-tracker-1.com/videos.js;a=123&b=abc", + "isAllowlisted": true + }, + { + "description": "should not remove port from url paths", + "site": "https://a.b.c.testsite.com", + "request": "https://allowlist-tracker-1.com/allowlist-tracker-1.com:5000/videos.js", + "isAllowlisted": false + }, + { + "description": "should match on all subdomains of an allowlisted tracker", + "site": "https://testsite.com", + "request": "https://a.b.c.allowlist-tracker-1.com/videos.js", + "isAllowlisted": true + }, + { + "description": "should not match on a substring of the domain", + "site": "https://anothertestsite.com", + "request": "https://allowlist-tracker-1.com/videos.js", + "isAllowlisted": false + }, + { + "description": "should not match on a site not listed in the allowlist entry domains list", + "site": "https://testsite2.com", + "request": "https://allowlist-tracker-1.com/videos.js", + "isAllowlisted": false + }, + { + "description": "should remove query strings from request", + "site": "https://testsite.com", + "request": "https://allowlist-tracker-1.com/videos.js?a=123&b=456", + "isAllowlisted": true + }, + { + "description": "should match random paths", + "site": "https://someothersite.com", + "request": "https://allowlist-tracker-2.com/comments/1234asdf/comment.js", + "isAllowlisted": true + }, + { + "description": "should match all requests for a whole domain rule", + "site": "https://testsite.com", + "request": "https://allowlist-tracker-3.com/tracker.js", + "isAllowlisted": true + }, + { + "description": "should match on specific subdomain rules", + "site": "https://testsite.com", + "request": "http://videos.allowlist-tracker-2.com/a.js", + "isAllowlisted": true + }, + { + "description": "should not match on root domain for a subdomain rule", + "site": "https://testsite.com", + "request": "http://allowlist-tracker-2.com/a.js", + "isAllowlisted": false + }, + { + "description": "should match on subdomain of a subdomain rule", + "site": "https://testsite.com", + "request": "http://a.b.c.videos.allowlist-tracker-2.com/a.js", + "isAllowlisted": true + } +] diff --git a/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_reference.json b/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_reference.json new file mode 100644 index 000000000..ea2866e55 --- /dev/null +++ b/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_reference.json @@ -0,0 +1,40 @@ +{ + "allowlist-tracker-1.com": { + "rules" : [ + { + "rule": "allowlist-tracker-1.com/videos.js", + "domains": ["testsite.com"], + "reason": "match single resource on single site" + } + ] + }, + "allowlist-tracker-2.com": { + "rules" : [ + { + "rule": "videos.allowlist-tracker-2.com/a.js", + "domains": ["testsite.com"], + "reason": "specific subdomain rule" + }, + { + "rule": "allowlist-tracker-2.com/comments/", + "domains": [""], + "reason": "match all sites and all paths" + }, + { + "rule": "allowlist-tracker-2.com/login.js", + "domains": [""], + "reason": "match single resource on all sites" + } + ] + }, + "allowlist-tracker-3.com": { + "rules": [ + { + "rule": "allowlist-tracker-3.com", + "domains": [""], + "reason": "match all requests" + } + ] + } +} + diff --git a/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_tds_reference.json b/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_tds_reference.json new file mode 100644 index 000000000..14f7f65df --- /dev/null +++ b/Tests/BrowserServicesKitTests/Resources/tracker_allowlist_tds_reference.json @@ -0,0 +1,65 @@ +{ + "trackers": { + "allowlist-tracker-1.com": { + "domain": "allowlist-tracker-1.com", + "owner": { + "name": "Test Site for Tracker Blocking", + "displayName": "allowlist-tracker-1.com", + "privacyPolicy": "", + "url": "http://allowlist-tracker-1.com" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "default": "block" + }, + "allowlist-tracker-2.com": { + "domain": "allowlist-tracker-2.com", + "owner": { + "name": "Test Site for Tracker Blocking", + "displayName": "allowlist-tracker-2.com", + "privacyPolicy": "", + "url": "http://allowlist-tracker-2.com" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "default": "block" + }, + "allowlist-tracker-3.com": { + "domain": "allowlist-tracker-3.com", + "owner": { + "name": "Test Site for Tracker Blocking", + "displayName": "allowlist-tracker-3.com", + "privacyPolicy": "", + "url": "http://allowlist-tracker-3.com" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "default": "block" + } + }, + "entities": { + "Test Site for Tracker Blocking": { + "domains": [ + "allowlist-tracker-1.com", + "allowlist-tracker-2.com", + "allowlist-tracker-3.com" + ], + "prevalence": 0.1, + "displayName": "Test Site for Tracker Blocking" + } + }, + "cnames": { + "bad.cnames.test": "allowlist-tracker-1.com" + }, + "domains": { + "allowlist-tracker-1.com": "Test Site for Tracker Blocking", + "allowlist-tracker-2.com": "Test Site for Tracker Blocking", + "allowlist-tracker-3.com": "Test Site for Tracker Blocking" + } +} diff --git a/Tests/BrowserServicesKitTests/Resources/tracker_radar_reference.json b/Tests/BrowserServicesKitTests/Resources/tracker_radar_reference.json new file mode 100644 index 000000000..f80c2e13b --- /dev/null +++ b/Tests/BrowserServicesKitTests/Resources/tracker_radar_reference.json @@ -0,0 +1,140 @@ +{ + "trackers": { + "bad.third-party.site": { + "domain": "bad.third-party.site", + "owner": { + "name": "Test Site for Tracker Blocking", + "displayName": "Bad Third Party Site", + "privacyPolicy": "", + "url": "http://bad.third-party.site" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "default": "block", + "rules": [ + { + "rule": "bad\\.third-party\\.site\\/ignore", + "action": "ignore" + }, + { + "rule": "bad\\.third-party\\.site", + "exceptions": { + "domains": [ + "ignore.test" + ], + "types": [ + "image" + ] + } + } + ] + }, + "broken.third-party.site": { + "domain": "broken.third-party.site", + "owner": { + "name": "Test Site for Tracker Blocking", + "displayName": "Broken Third Party Site", + "privacyPolicy": "", + "url": "http://broken.third-party.site" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "rules": [], + "default": "ignore" + }, + "tracker.test": { + "domain": "tracker.test", + "owner": { + "name": "Test Site for Tracker Blocking", + "displayName": "Bad Third Party Site", + "privacyPolicy": "", + "url": "http://tracker.test" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "default": "block" + }, + "ignore.test": { + "domain": "ignore.test", + "owner": { + "name": "Test Site for Tracker Blocking", + "displayName": "ignore Site", + "privacyPolicy": "", + "url": "http://ignore.test" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "default": "ignore", + "rules": [ + { + "rule": "ignore\\.test\\/tracker" + } + ] + }, + "surrogates.test": { + "domain": "surrogates.test", + "owner": { + "name": "Test Site for surrogates", + "displayName": "Bad Third Party Site", + "privacyPolicy": "", + "url": "http://surrogates.test" + }, + "prevalence": 0.1, + "fingerprinting": 3, + "cookies": 0.1, + "categories": [], + "rules": [ + { + "rule": "surrogates\\.test\\/tracker", + "surrogate": "tracker" + }, + { + "rule": "surrogates\\.test\\/anothertracker", + "surrogate": "missingsurrogate" + } + ], + "default": "ignore" + } + }, + "entities": { + "Test Site for Tracker Blocking": { + "domains": [ + "bad.third-party.site", + "broken.third-party.site", + "third-party.site", + "tracker.test", + "surrogates.test" + ], + "prevalence": 0.1, + "displayName": "Test Site for Tracker Blocking" + }, + "ignore Site for Tracker Blocking": { + "domains": [ + "ignore.test", + "sub.ignore.test" + ], + "prevalence": 0.1, + "displayName": "ignore Site for Tracker Blocking" + } + }, + "cnames": { + "bad.cnames.test": "cname.tracker.test" + }, + "domains": { + "bad.third-party.site": "Test Site for Tracker Blocking", + "broken.third-party.site": "Test Site for Tracker Blocking", + "third-party.site": "Test Site for Tracker Blocking", + "tracker.test": "Test Site for Tracker Blocking", + "ignore.test": "ignore Site for Tracker Blocking", + "sub.ignore.test": "ignore Site for Tracker Blocking", + "surrogates.test": "Test Site for surrogates" + } +} diff --git a/Tests/BrowserServicesKitTests/SecureVault/MockVaultProviders.swift b/Tests/BrowserServicesKitTests/SecureVault/MockVaultProviders.swift index 902b08f60..dfcd787bd 100644 --- a/Tests/BrowserServicesKitTests/SecureVault/MockVaultProviders.swift +++ b/Tests/BrowserServicesKitTests/SecureVault/MockVaultProviders.swift @@ -23,9 +23,15 @@ internal class MockDatabaseProvider: SecureVaultDatabaseProvider { // swiftlint:disable identifier_name var _accounts = [SecureVaultModels.WebsiteAccount]() + var _notes = [SecureVaultModels.Note]() + var _identities = [SecureVaultModels.Identity]() + var _creditCards = [SecureVaultModels.CreditCard]() var _forDomain = [String]() var _credentials: SecureVaultModels.WebsiteCredentials? var _lastCredentials: SecureVaultModels.WebsiteCredentials? + var _note: SecureVaultModels.Note? + var _identity: SecureVaultModels.Identity? + var _creditCard: SecureVaultModels.CreditCard? // swiftlint:enable identifier_name func storeWebsiteCredentials(_ credentials: SecureVaultModels.WebsiteCredentials) throws -> Int64 { @@ -50,6 +56,56 @@ internal class MockDatabaseProvider: SecureVaultDatabaseProvider { return _accounts } + func notes() throws -> [SecureVaultModels.Note] { + return _notes + } + + func noteForNoteId(_ noteId: Int64) throws -> SecureVaultModels.Note? { + return _note + } + + func deleteNoteForNoteId(_ noteId: Int64) throws { + self._notes = self._notes.filter { $0.id != noteId } + } + + func storeNote(_ note: SecureVaultModels.Note) throws -> Int64 { + _note = note + return note.id ?? -1 + } + + func identities() throws -> [SecureVaultModels.Identity] { + return _identities + } + + func identityForIdentityId(_ identityId: Int64) throws -> SecureVaultModels.Identity? { + return _identity + } + + func storeIdentity(_ identity: SecureVaultModels.Identity) throws -> Int64 { + _identity = identity + return identity.id ?? -1 + } + + func deleteIdentityForIdentityId(_ identityId: Int64) throws { + self._identities = self._identities.filter { $0.id != identityId } + } + + func creditCards() throws -> [SecureVaultModels.CreditCard] { + return _creditCards + } + + func creditCardForCardId(_ cardId: Int64) throws -> SecureVaultModels.CreditCard? { + return _creditCard + } + + func storeCreditCard(_ creditCard: SecureVaultModels.CreditCard) throws -> Int64 { + _creditCard = creditCard + return creditCard.id ?? -1 + } + + func deleteCreditCardForCreditCardId(_ cardId: Int64) throws { + self._creditCards = self._creditCards.filter { $0.id != cardId } + } } internal class MockCryptoProvider: SecureVaultCryptoProvider { diff --git a/Tests/BrowserServicesKitTests/Suggestions/SuggestionTests.swift b/Tests/BrowserServicesKitTests/Suggestions/SuggestionTests.swift index 9f103a6a2..188c0790b 100644 --- a/Tests/BrowserServicesKitTests/Suggestions/SuggestionTests.swift +++ b/Tests/BrowserServicesKitTests/Suggestions/SuggestionTests.swift @@ -22,13 +22,13 @@ import XCTest final class SuggestionTests: XCTestCase { func testSuggestionInitializedFromBookmark() { - let url = URL(string: "duckduckgo.com")! + let url = URL.aURL let title = "DuckDuckGo" let isFavorite = true let bookmarkMock = BookmarkMock(url: url, title: title, isFavorite: isFavorite) let suggestion = Suggestion(bookmark: bookmarkMock) - XCTAssertEqual(suggestion, Suggestion.bookmark(title: title, url: url, isFavorite: isFavorite)) + XCTAssertEqual(suggestion, Suggestion.bookmark(title: title, url: url, isFavorite: isFavorite, allowedInTopHits: isFavorite)) } func testWhenSuggestionKeyIsPhrase_ThenSuggestionIsPhrase() { @@ -48,12 +48,14 @@ final class SuggestionTests: XCTestCase { } func testWhenUrlIsAccessed_ThenOnlySuggestionsThatContainUrlReturnsIt() { - let url = URL(string: "https://www.duckduckgo.com")! + let url = URL.aURL let phraseSuggestion = Suggestion.phrase(phrase: "phrase") let websiteSuggestion = Suggestion.website(url: url) - let bookmarkSuggestion = Suggestion.bookmark(title: "Title", url: url, isFavorite: true) - let historyEntrySuggestion = Suggestion.historyEntry(title: "Title", url: url, allowedInTopHits: true) + let bookmarkSuggestion = Suggestion.bookmark(title: "Title", url: url, isFavorite: true, allowedInTopHits: true) + let historyEntrySuggestion = Suggestion.historyEntry(title: "Title", + url: url, + allowedInTopHits: true) _ = Suggestion.unknown(value: "phrase") XCTAssertNil(phraseSuggestion.url) @@ -64,12 +66,12 @@ final class SuggestionTests: XCTestCase { } func testWhenTitleIsAccessed_ThenOnlySuggestionsThatContainUrlStoreIt() { - let url = URL(string: "https://www.duckduckgo.com")! + let url = URL.aURL let title = "Original Title" let phraseSuggestion = Suggestion.phrase(phrase: "phrase") let websiteSuggestion = Suggestion.website(url: url) - let bookmarkSuggestion = Suggestion.bookmark(title: title, url: url, isFavorite: true) + let bookmarkSuggestion = Suggestion.bookmark(title: title, url: url, isFavorite: true, allowedInTopHits: true) let historyEntrySuggestion = Suggestion.historyEntry(title: title, url: url, allowedInTopHits: true) _ = Suggestion.unknown(value: "phrase") @@ -81,7 +83,7 @@ final class SuggestionTests: XCTestCase { } func testWhenInitFromHistoryEntry_ThenHistroryEntrySuggestionIsInitialized() { - let url = URL(string: "https://www.duckduckgo.com")! + let url = URL.aURL let title = "Title" @@ -98,7 +100,7 @@ final class SuggestionTests: XCTestCase { } func testWhenInitFromBookmark_ThenBookmarkSuggestionIsInitialized() { - let url = URL(string: "https://www.duckduckgo.com")! + let url = URL.aURL let title = "Title" @@ -115,7 +117,7 @@ final class SuggestionTests: XCTestCase { } func testWhenInitFromURL_ThenWebsiteSuggestionIsInitialized() { - let url = URL(string: "https://www.duckduckgo.com")! + let url = URL.aURL let suggestion = Suggestion(url: url) guard case .website(let websiteUrl) = suggestion else { @@ -127,4 +129,61 @@ final class SuggestionTests: XCTestCase { XCTAssertEqual(websiteUrl, url) } + func testWhenSuggestionIsWebsite_ThenCanBeInTopHits() { + let suggestion = Suggestion.website(url: .aURL) + XCTAssert(suggestion.allowedInTopHits) + } + + func testWhenSuggestionIsLowVisitNonRootHistoryEntry_ThenCantBeInTopHits() { + let historyEntry = HistoryEntryMock(identifier: UUID(), + url: .aNonRootUrl, + title: "Title", + numberOfVisits: 1, + lastVisit: Date(), + failedToLoad: false, + isDownload: false) + let suggestion = Suggestion(historyEntry: historyEntry) + XCTAssertFalse(suggestion.allowedInTopHits) + } + + func testWhenSuggestionIsLowVisitRootHistoryEntry_ThenCanBeInTopHits() { + let historyEntry = HistoryEntryMock(identifier: UUID(), + url: .aRootUrl, + title: "Title", + numberOfVisits: 1, + lastVisit: Date(), + failedToLoad: false, + isDownload: false) + let suggestion = Suggestion(historyEntry: historyEntry) + XCTAssert(suggestion.allowedInTopHits) + } + + func testWhenSuggestionIsFailingLink_ThenCantBeInTopHits() { + let historyEntry = HistoryEntryMock(identifier: UUID(), + url: .aURL, + title: "Title", + numberOfVisits: 100, + lastVisit: Date(), + failedToLoad: true, + isDownload: false) + let suggestion = Suggestion(historyEntry: historyEntry) + XCTAssertFalse(suggestion.allowedInTopHits) + } + + func testWhenSuggestionIsBookmark_ThenCanOrCantBeInTopHits() { + let suggestion = Suggestion.bookmark(title: "Title", url: .aURL, isFavorite: false, allowedInTopHits: false) + XCTAssertFalse(suggestion.allowedInTopHits) + + let suggestion2 = Suggestion.bookmark(title: "Title", url: .aURL, isFavorite: false, allowedInTopHits: true) + XCTAssert(suggestion2.allowedInTopHits) + } + +} + +fileprivate extension URL { + + static let aURL = URL(string: "https://www.duckduckgo.com")! + static let aRootUrl = aURL + static let aNonRootUrl = URL(string: "https://www.duckduckgo.com/traffic")! + } diff --git a/Tests/BrowserServicesKitTests/UserScript/ContentScopePropertiesTests.swift b/Tests/BrowserServicesKitTests/UserScript/ContentScopePropertiesTests.swift new file mode 100644 index 000000000..a8d21596f --- /dev/null +++ b/Tests/BrowserServicesKitTests/UserScript/ContentScopePropertiesTests.swift @@ -0,0 +1,34 @@ +// +// ContentScopePropertiesTests.swift +// DuckDuckGo +// +// Copyright © 2021 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import WebKit +@testable import BrowserServicesKit + +class ContentScopePropertiesTests: XCTestCase { + func testContentScopePropertiesInitializeCorrectly() { + let properties = ContentScopeProperties(gpcEnabled: true, sessionKey: "123456"); + + // ensure the properties can be encoded to valid JSON + XCTAssertNotNil(try? JSONEncoder().encode(properties)) + + // ensure the platform.name key exists, as this will be expected in the output JSON + XCTAssertEqual(properties.platform.name, ContentScopePlatform().name) + } +} diff --git a/Tests/BrowserServicesKitTests/Utils/JsonTestDataLoader.swift b/Tests/BrowserServicesKitTests/Utils/JsonTestDataLoader.swift new file mode 100644 index 000000000..549fd9622 --- /dev/null +++ b/Tests/BrowserServicesKitTests/Utils/JsonTestDataLoader.swift @@ -0,0 +1,71 @@ +// +// JsonTestDataLoader.swift +// DuckDuckGo +// +// Copyright © 2017 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +enum FileError: Error { + case unknownFile + case invalidFileContents +} + +final class FileLoader { + + func load(fileName: String, fromBundle bundle: Bundle) throws -> Data { + + let fileUrl = URL(fileURLWithPath: fileName) + let baseName = fileUrl.deletingPathExtension().lastPathComponent + let ext = fileUrl.pathExtension + + guard let path = bundle.path(forResource: baseName, ofType: ext) else { throw FileError.unknownFile } + let url = URL(fileURLWithPath: path) + guard let data = try? Data(contentsOf: url, options: [.mappedIfSafe]) else { throw FileError.invalidFileContents } + return data + } +} + +final class JsonTestDataLoader { + + func empty() -> Data { + return "".data(using: .utf16)! + } + + func invalid() -> Data { + return "{[}".data(using: .utf16)! + } + + func unexpected() -> Data { + guard let data = try? FileLoader().load(fileName: "MockFiles/unexpected.json", fromBundle: bundle) else { + fatalError("Failed to load MockFiles/unexpected.json") + } + return data + } + + func fromJsonFile(_ fileName: String) -> Data { + + do { + return try FileLoader().load(fileName: fileName, fromBundle: bundle) + } catch { + fatalError("Unable to load \(fileName) error \(error)") + } + } + + private var bundle: Bundle { + return Bundle.module + } +}