Add Phishing Detection Package (#935)

Please review the release process for BrowserServicesKit [here](https://app.asana.com/0/1200194497630846/1200837094583426). **Required**: Task/Issue URL: https://app.asana.com/0/1204023833050360/1207976613228509/f iOS PR: duckduckgo/iOS#3336 macOS PR: duckduckgo/macos-browser#3206 What kind of version bump will this require?: Minor **Optional**: Tech Design URL: https://app.asana.com/0/481882893211075/1207156899292810/f CC: https://app.asana.com/0/481882893211075/1207220724600204/f **Description**: Implement Phishing Detection library to facilitate end-to-end phishing detection feature. Including: 1. Background data updates 2. API client for updating data 3. Embedded datasets 4. Detection logic 5. Event firing  **Steps to test this PR**: 1. Build on macOS 2. Ensure signed in via use-login.duckduckgo.com (only available internally) 3. Quit the app 4. Visit https://privacy-test-pages.site/security/badware/phishing.html 5. Ensure error page is thrown 6. Click advanced 7. Click "Accept Risk" 8. Ensure page loads 9. Play around with navigations (back/forward, etc.) 10. Disable the feature in Settings>General 11. Try other test pages: - https://bad.third-party.site/security/badware/phishing-iframe-loader.html - https://bad.third-party.site/security/badware/phishing-meta-redirect.html - https://bad.third-party.site/security/badware/phishing-js-redirector-helper.html  **OS Testing**: * [ ] iOS 14 * [ ] iOS 15 * [ ] iOS 16 * [ ] macOS 10.15 * [ ] macOS 11 * [ ] macOS 12 --- ###### Internal references: [Software Engineering Expectations](https://app.asana.com/0/59792373528535/199064865822552) [Technical Design Template](https://app.asana.com/0/59792373528535/184709971311943) --------- Co-authored-by: Sabrina Tardio <[email protected]>
duckduckgo · Sep 9, 2024 · ca2c103 · ca2c103
1 parent b83ccf1
commit ca2c103
Show file tree

Hide file tree

Showing 35 changed files with 2,149 additions and 8 deletions.
diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit.xcscheme
@@ -164,6 +164,16 @@
                ReferencedContainer = "container:">
             </BuildableReference>
          </TestableReference>
+         <TestableReference
+             skipped = "NO">
+             <BuildableReference
+                BuildableIdentifier = "primary"
+                BlueprintIdentifier = "PhishingDetectionTests"
+                BuildableName = "PhishingDetectionTests"
+                BlueprintName = "PhishingDetectionTests"
+                ReferencedContainer = "container:">
+             </BuildableReference>
+          </TestableReference>
       </Testables>
    </TestAction>
    <LaunchAction

diff --git a/Package.swift b/Package.swift
@@ -42,6 +42,7 @@ let package = Package(
         .library(name: "PixelKitTestingUtilities", targets: ["PixelKitTestingUtilities"]),
         .library(name: "SpecialErrorPages", targets: ["SpecialErrorPages"]),
         .library(name: "DuckPlayer", targets: ["DuckPlayer"]),
+        .library(name: "PhishingDetection", targets: ["PhishingDetection"]),
         .library(name: "Onboarding", targets: ["Onboarding"])
     ],
     dependencies: [
@@ -50,8 +51,8 @@ let package = Package(
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.2.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "2.1.0"),
+        .package(url: "https://github.com/duckduckgo/privacy-dashboard", branch:"mgurgel/phishing-warning"),
         .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "6.14.1"),
-        .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "5.1.1"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
         .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1")
@@ -403,6 +404,19 @@ let package = Package(
                 .define("DEBUG", .when(configuration: .debug))
             ]
         ),
+        .target(
+            name: "PhishingDetection",
+            dependencies: [
+                "Common"
+            ],
+            resources: [
+                .copy("hashPrefixes.json"),
+                .copy("filterSet.json")
+            ],
+            swiftSettings: [
+                .define("DEBUG", .when(configuration: .debug))
+            ]
+        ),
         .target(
             name: "Onboarding",
             dependencies: [
@@ -613,6 +627,18 @@ let package = Package(
                 "DuckPlayer"
             ]
         ),
+
+        .testTarget(
+            name: "PhishingDetectionTests",
+            dependencies: [
+                "PhishingDetection",
+                "PixelKit"
+            ],
+            resources: [
+                .copy("hashPrefixes.json"),
+                .copy("filterSet.json")
+            ]
+        ),
         .testTarget(
             name: "OnboardingTests",
             dependencies: [

diff --git a/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift b/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift
@@ -49,6 +49,7 @@ public enum PrivacyFeature: String {
     case sslCertificates
     case brokenSiteReportExperiment
     case toggleReports
+    case phishingDetection
     case brokenSitePrompt
     case remoteMessaging
     case additionalCampaignPixelParams
@@ -124,7 +125,7 @@ public enum PrivacyProSubfeature: String, Equatable, PrivacySubfeature {
     case useUnifiedFeedback
 }
 
-public enum sslCertificatesSubfeature: String, PrivacySubfeature {
+public enum SslCertificatesSubfeature: String, PrivacySubfeature {
     public var parent: PrivacyFeature { .sslCertificates }
     case allowBypass
 }
@@ -136,6 +137,12 @@ public enum DuckPlayerSubfeature: String, PrivacySubfeature {
     case openInNewTab
 }
 
+public enum PhishingDetectionSubfeature: String, PrivacySubfeature {
+    public var parent: PrivacyFeature { .phishingDetection }
+    case allowErrorPage
+    case allowPreferencesToggle
+}
+
 public enum SyncPromotionSubfeature: String, PrivacySubfeature {
     public var parent: PrivacyFeature { .syncPromotion }
     case bookmarks

diff --git a/Sources/Common/Extensions/URLExtension.swift b/Sources/Common/Extensions/URLExtension.swift
@@ -442,6 +442,104 @@ extension URL {
         return host == protectionSpace.host && (port ?? navigationalScheme?.defaultPort) == protectionSpace.port && scheme == protectionSpace.protocol
     }
 
+    // MARK: Canonicalization 
+    public func canonicalHost() -> String? {
+        // Step 1: Extract hostname portion from the URL
+        guard var canonicalHost = self.host else {
+            return nil
+        }
+
+        // Step 2: Decode any %XX escapes present in the hostname
+        if let decodedHost = canonicalHost.removingPercentEncoding {
+            canonicalHost = decodedHost
+        }
+
+        // Step 3: Discard any characters outside the range 0x20 to 0x7E
+        canonicalHost = canonicalHost.filter { character in
+            let asciiValue = character.unicodeScalars.first?.value ?? 0
+            return (asciiValue >= 0x20 && asciiValue <= 0x7E)
+        }
+
+        // Step 4: Discard any leading and/or trailing full-stops
+        canonicalHost = canonicalHost.trimmingCharacters(in: CharacterSet(charactersIn: "."))
+
+        // Step 5: Replace sequences of two or more full-stops with a single full-stop
+        canonicalHost = canonicalHost.replacingOccurrences(of: "\\.+", with: ".", options: .regularExpression)
+
+        // Step 6: If the hostname is a numeric IPv4 address then reduce it to the canonical dotted quad form
+        let ipv4AddressComponents = canonicalHost.components(separatedBy: ".")
+        if ipv4AddressComponents.count == 4, ipv4AddressComponents.allSatisfy({ Int($0) != nil }) {
+            canonicalHost = ipv4AddressComponents.joined(separator: ".")
+        }
+
+        // Step 7: Replace any characters other than letters, numbers, ".", and "-" with "%XX" escape codes, using lowercase hexadecimal digits
+        canonicalHost = canonicalHost.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) ?? ""
+
+        // Step 8: If more than six components in the resulting hostname, discard all but the rightmost six components
+        let components = canonicalHost.components(separatedBy: ".").suffix(6)
+        canonicalHost = components.joined(separator: ".")
+
+        return canonicalHost
+     }
+
+    public func canonicalURL() -> URL? {
+        // Step 1: Remove tab (0x09), CR (0x0d), and LF (0x0a) characters
+        var urlString = self.absoluteString.filter { $0 != "\t" && $0 != "\r" && $0 != "\n" }
+
+        // Step 2: Remove the fragment
+        if let fragmentRange = urlString.range(of: "#") {
+            urlString.removeSubrange(fragmentRange.lowerBound..<urlString.endIndex)
+        }
+
+        // Step 3: Repeatedly percent-unescape the URL until it has no more percent-escapes
+        var previousURLString: String
+        repeat {
+            previousURLString = urlString
+            if let unescapedURLString = urlString.removingPercentEncoding {
+                urlString = unescapedURLString
+            }
+        } while urlString != previousURLString
+
+        // Step 4: Remove all trailing slashes, but keep the single slash after the domain
+        if let url = URL(string: urlString), url.path == "/" {
+            // Do not remove the single trailing slash if it's just the domain
+        } else {
+            while urlString.last == "/" {
+                urlString.removeLast()
+            }
+        }
+
+        // Step 5: Remove all occurrences of more than one "/", but not in the protocol part
+        if let range = urlString.range(of: "://") {
+            let protocolPart = urlString[..<range.upperBound]
+            let restOfURL = urlString[range.upperBound...]
+            urlString = protocolPart + restOfURL.replacingOccurrences(of: "/+", with: "/", options: .regularExpression)
+        }
+
+        // Step 6: Remove all occurrences of "/./" in the path
+        urlString = urlString.replacingOccurrences(of: "/./", with: "/")
+
+        // Step 7: Remove all occurrences of "/../" in the path
+        while let range = urlString.range(of: "/../") {
+            let previousComponentRange = urlString.range(of: "/", options: .backwards, range: urlString.startIndex..<range.lowerBound)
+            if let previousComponentRange = previousComponentRange {
+                urlString.removeSubrange(previousComponentRange.upperBound..<range.upperBound)
+            } else {
+                break
+            }
+        }
+
+        // Step 8: Lowercase everything
+        urlString = urlString.lowercased()
+
+        // Validate the URL according to RFC 2396
+        guard let validURL = URL(string: urlString), validURL.path.count > 0 else {
+            return nil
+        }
+
+        return validURL
+    }
+
 }
 
 public extension CharacterSet {

diff --git a/Sources/Navigation/FrameHandle.swift b/Sources/Navigation/FrameHandle.swift
@@ -32,7 +32,7 @@ public struct FrameHandle: Hashable, _ObjectiveCBridgeable {
         return object.value(forKey: Self.frameIDKey) as? UInt64 ?? 0
     }
 
-    init?(rawValue: Any) {
+    public init?(rawValue: Any) {
         guard rawValue is UInt64
                 || (rawValue as? NSObject)?.responds(to: NSSelectorFromString("_" + Self.frameIDKey)) == true
                 || (rawValue as? NSObject)?.responds(to: NSSelectorFromString(Self.frameIDKey)) == true

diff --git a/Sources/PhishingDetection/Logger+PhishingDetection.swift b/Sources/PhishingDetection/Logger+PhishingDetection.swift
@@ -0,0 +1,29 @@
+//
+//  Logger+PhishingDetection.swift
+//
+//  Copyright © 2023 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+import os
+
+public extension Logger {
+    static var phishingDetection: Logger = { Logger(subsystem: "Phishing Detection", category: "") }()
+    static var phishingDetectionClient: Logger = { Logger(subsystem: "Phishing Detection", category: "APIClient") }()
+    static var phishingDetectionTasks: Logger = { Logger(subsystem: "Phishing Detection", category: "BackgroundActivities") }()
+    static var phishingDetectionDataProvider: Logger = { Logger(subsystem: "Phishing Detection", category: "DataProvider") }()
+    static var phishingDetectionDataStore: Logger = { Logger(subsystem: "Phishing Detection", category: "DataStore") }()
+    static var phishingDetectionUpdateManager: Logger = { Logger(subsystem: "Phishing Detection", category: "UpdateManager") }()
+}