Fix #905: Regional Adblock implementation (#672)

BraveShared/Preferences.swift

@@ -71,8 +71,9 @@ extension Preferences {
         public static let fingerprintingProtection = Option<Bool>(key: "shields.fingerprinting-protection", default: false)
         /// Disables image loading in the browser
         public static let blockImages = Option<Bool>(key: "shields.block-images", default: false)
-        ///
-        public static let useRegionAdBlock = Option<Bool>(key: "shields.regional-adblock", default: false)
+        /// In addition to global adblocking rules, adds custom country based rules.
+        /// This setting is enabled by default for all locales.
+        public static let useRegionAdBlock = Option<Bool>(key: "shields.regional-adblock", default: true)
         /// Version of downloaded data file for adblock stats.
         public static let adblockStatsDataVersion = Option<Int?>(key: "stats.adblock-data-version", default: nil)
     }

Client/Application/AppDelegate.swift

@@ -61,6 +61,8 @@ class AppDelegate: UIResponder, UIApplicationDelegate, UIViewControllerRestorati
         self.window!.backgroundColor = UIColor.Photon.White100
 
         AdBlockStats.shared.startLoading()
+        AdblockResourceDownloader.shared.regionalAdblockResourcesSetup()
+
         HttpsEverywhereStats.shared.startLoading()
 
         // Passcode checking, must happen on immediate launch

Client/FileManagerExtension.swift

@@ -4,6 +4,7 @@
 
 import Foundation
 import Shared
+import Deferred
 
 private let log = Logger.browserLogger
 
@@ -40,4 +41,44 @@ public extension FileManager {
         }
         return false
     }
+
+    func writeToDiskInFolder(_ data: Data, fileName: String, folderName: String) -> Bool {
+
+        guard let folderUrl = getOrCreateFolder(name: folderName) else { return false }
+
+        do {
+            let fileUrl = folderUrl.appendingPathComponent(fileName)
+            try data.write(to: fileUrl, options: [.atomic])
+        } catch {
+            log.error("Failed to write data, error: \(error)")
+            return false
+        }
+
+        return true
+    }
+
+    /// Creates a folder at documents directory and returns its URL.
+    /// If folder already exists, returns its URL as well.
+    func getOrCreateFolder(name: String, excludeFromBackups: Bool = true) -> URL? {
+        guard let documentsDir = urls(for: .documentDirectory, in: .userDomainMask).first else { return nil }
+
+        var folderDir = documentsDir.appendingPathComponent(name)
+
+        if fileExists(atPath: folderDir.path) { return folderDir }
+
+        do {
+            try createDirectory(at: folderDir, withIntermediateDirectories: true, attributes: nil)
+
+            if excludeFromBackups {
+                var resourceValues = URLResourceValues()
+                resourceValues.isExcludedFromBackup = true
+                try folderDir.setResourceValues(resourceValues)
+            }
+
+            return folderDir
+        } catch {
+            log.error("Failed to create folder, error: \(error)")
+            return nil
+        }
+    }
 }

Client/Frontend/Browser/BrowserViewController.swift

@@ -188,7 +188,7 @@ class BrowserViewController: UIViewController {
         Preferences.Shields.allShields.forEach { $0.observe(from: self) }
         Preferences.Privacy.blockAllCookies.observe(from: self)
         // Lists need to be compiled before attempting tab restoration
-        contentBlockListDeferred = ContentBlockerHelper.compileLists()
+        contentBlockListDeferred = ContentBlockerHelper.compileBundledLists()
     }
 
     override var preferredStatusBarStyle: UIStatusBarStyle {
@@ -2922,7 +2922,8 @@ extension BrowserViewController: PreferencesObserver {
              Preferences.Shields.blockScripts.key,
              Preferences.Shields.blockPhishingAndMalware.key,
              Preferences.Shields.blockImages.key,
-             Preferences.Shields.fingerprintingProtection.key:
+             Preferences.Shields.fingerprintingProtection.key,
+             Preferences.Shields.useRegionAdBlock.key:
             tabManager.allTabs.forEach { $0.webView?.reload() }
         case Preferences.Privacy.blockAllCookies.key:
             // All `block all cookies` toggle requires a hard reset of Webkit configuration.

Client/Frontend/ContentBlocker/BlocklistName.swift

@@ -1,92 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import WebKit
-import Shared
-import Deferred
-import Data
-import BraveShared
-
-private let log = Logger.browserLogger
-
-class BlocklistName: Hashable, CustomStringConvertible, ContentBlocker {
-
-    static let ad = BlocklistName(filename: "block-ads")
-    static let tracker = BlocklistName(filename: "block-trackers")
-    static let https = BlocklistName(filename: "upgrade-http")
-    static let image = BlocklistName(filename: "block-images")
-    static let cookie = BlocklistName(filename: "block-cookies")
-
-    static var allLists: Set<BlocklistName> { return [.ad, .tracker, .https, .image] }
-
-    let filename: String
-    var rule: WKContentRuleList?
-
-    init(filename: String) {
-        self.filename = filename
-    }
-
-    var description: String {
-        return "<\(type(of: self)): \(self.filename)>"
-    }
-
-    private static let blocklistFileVersionMap: [BlocklistName: Preferences.Option<String?>] = [
-        BlocklistName.ad: Preferences.BlockFileVersion.adblock,
-        BlocklistName.https: Preferences.BlockFileVersion.httpse
-    ]
-
-    lazy var fileVersionPref: Preferences.Option<String?>? = {
-        return BlocklistName.blocklistFileVersionMap[self]
-    }()
-
-    lazy var fileVersion: String? = {
-        guard let _ = BlocklistName.blocklistFileVersionMap[self] else { return nil }
-        return Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String
-    }()
-
-    static func blocklists(forDomain domain: Domain) -> (on: Set<BlocklistName>, off: Set<BlocklistName>) {
-        if domain.shield_allOff == 1 {
-            return ([], allLists)
-        }
-
-        var onList = Set<BlocklistName>()
-
-        let isPrivateBrowsing = PrivateBrowsingManager.shared.isPrivateBrowsing
-        if domain.isShieldExpected(.AdblockAndTp, isPrivateBrowsing: isPrivateBrowsing) {
-            onList.formUnion([.ad, .tracker])
-        }
-
-        // For lists not implemented, always return exclude from `onList` to prevent accidental execution
-
-        // TODO #159: Setup image shield
-
-        if domain.isShieldExpected(.HTTPSE, isPrivateBrowsing: isPrivateBrowsing) {
-            onList.formUnion([.https])
-        }
-
-        return (onList, allLists.subtracting(onList))
-    }
-
-    static func compileAll(ruleStore: WKContentRuleListStore) -> Deferred<Void> {
-        let allCompiledDeferred = Deferred<Void>()
-        var allOfThem = BlocklistName.allLists.map {
-            $0.buildRule(ruleStore: ruleStore)
-        }
-        //Compile block-cookie additionally 
-        allOfThem.append(BlocklistName.cookie.buildRule(ruleStore: ruleStore))
-        all(allOfThem).upon { _ in
-            allCompiledDeferred.fill(())
-        }
-
-        return allCompiledDeferred
-    }
-
-    public static func == (lhs: BlocklistName, rhs: BlocklistName) -> Bool {
-        return lhs.filename == rhs.filename
-    }
-
-    public func hash(into hasher: inout Hasher) {
-        hasher.combine(filename)
-    }
-}

Client/Frontend/Settings/SettingsViewController.swift

@@ -295,7 +295,7 @@ class SettingsViewController: TableViewController {
     }()
 
     private lazy var shieldsSection: Section = {
-        return Section(
+        var shields = Section(
             header: .title(Strings.Brave_Shield_Defaults),
             rows: [
                 BoolRow(title: Strings.Block_Ads_and_Tracking, option: Preferences.Shields.blockAdsAndTracking),
@@ -305,8 +305,10 @@ class SettingsViewController: TableViewController {
                 BoolRow(title: Strings.Fingerprinting_Protection, option: Preferences.Shields.fingerprintingProtection),
             ]
         )
-        // TODO: Add regional adblock
-        // shields.rows.append(BasicBoolRow(title: Strings.Use_regional_adblock, option: Preferences.Shields.useRegionAdBlock))
+        if let locale = Locale.current.languageCode, let _ = ContentBlockerRegion.with(localeCode: locale) {
+            shields.rows.append(BoolRow(title: Strings.Use_regional_adblock, option: Preferences.Shields.useRegionAdBlock))
+        }
+        return shields
     }()
 
     private lazy var supportSection: Section = {
@@ -380,7 +382,7 @@ class SettingsViewController: TableViewController {
                 Row(text: "Region: \(Locale.current.regionCode ?? "--")"),
                 Row(text: "Recompile Content Blockers", selection: { [weak self] in
                     BlocklistName.allLists.forEach { $0.fileVersionPref?.value = nil }
-                    ContentBlockerHelper.compileLists().upon {
+                    ContentBlockerHelper.compileBundledLists().upon { _ in
                         let alert = UIAlertController(title: nil, message: "Recompiled Blockers", preferredStyle: .alert)
                         alert.addAction(UIAlertAction(title: "OK", style: .default))
                         self?.present(alert, animated: true)

Client/Networking/CachedNetworkResource.swift

@@ -0,0 +1,11 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import Foundation
+
+struct CachedNetworkResource {
+    let data: Data
+    let etag: String?
+}
+

Client/Networking/NetworkManager.swift

@@ -0,0 +1,89 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import Foundation
+import Deferred
+import Shared
+
+private let log = Logger.browserLogger
+
+class NetworkManager {
+    private let session: NetworkSession
+
+    init(session: NetworkSession = URLSession.shared) {
+        self.session = session
+    }
+
+    func dataRequest(with url: URL, completion: @escaping NetworkSessionDataResponse) {
+        session.dataRequest(with: url) { data, response, error in
+            completion(data, response, error)
+        }
+    }
+
+    func dataRequest(with urlRequest: URLRequest, completion: @escaping NetworkSessionDataResponse) {
+        session.dataRequest(with: urlRequest) { data, response, error in
+            completion(data, response, error)
+        }
+    }
+
+    func downloadResource(with url: URL, resourceType: NetworkResourceType,
+                          retryTimeout: TimeInterval? = 60) -> Deferred<CachedNetworkResource> {
+        let completion = Deferred<CachedNetworkResource>()
+
+        var request = URLRequest(url: url)
+
+        // Makes the request conditional, returns 304 if Etag value did not change.
+        let ifNoneMatchHeader = "If-None-Match"
+        let fileNotModifiedStatusCode = 304
+
+        // Identifier for a specific version of a resource for a HTTP request
+        let etagHeader = "Etag"
+
+        switch resourceType {
+        case .cached(let etag):
+            let requestEtag = etag ?? UUID().uuidString
+
+            // This cache policy is required to support `If-None-Match` header.
+            request.cachePolicy = .reloadIgnoringLocalAndRemoteCacheData
+            request.addValue(requestEtag, forHTTPHeaderField: ifNoneMatchHeader)
+        default: break
+        }
+
+        session.dataRequest(with: request) { data, response, error -> Void in
+            if let err = error {
+                log.error(err.localizedDescription)
+                if let retryTimeout = retryTimeout {
+                    DispatchQueue.main.asyncAfter(deadline: .now() + retryTimeout) {
+                        self.downloadResource(with: url, resourceType: resourceType, retryTimeout: retryTimeout).upon { resource in
+                            completion.fill(resource)
+                        }
+                    }
+                }
+                return
+            }
+
+            guard let data = data, let response = response as? HTTPURLResponse else {
+                log.error("Failed to unwrap http response or data")
+                return
+            }
+
+            switch response.statusCode {
+            case 400...499:
+                log.error("""
+                    Failed to download, status code: \(response.statusCode),\
+                    URL:\(String(describing: response.url))
+                    """)
+            case fileNotModifiedStatusCode:
+                log.info("File not modified")
+            default:
+                let responseEtag = resourceType.isCached() ?
+                    response.allHeaderFields[etagHeader] as? String : nil
+
+                completion.fill(CachedNetworkResource(data: data, etag: responseEtag))
+            }
+        }
+
+        return completion
+    }
+}

Client/Networking/NetworkResourceType.swift

@@ -0,0 +1,17 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import Foundation
+
+enum NetworkResourceType {
+    case cached(etag: String?)
+    case regular
+
+    func isCached() -> Bool {
+        switch self {
+        case .cached(_): return true
+        default: return false
+        }
+    }
+}

Client/Networking/NetworkSession.swift

@@ -0,0 +1,30 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import Foundation
+
+typealias NetworkSessionDataResponse = (Data?, URLResponse?, Error?) -> Void
+
+protocol NetworkSession {
+    func dataRequest(with url: URL, completion: @escaping NetworkSessionDataResponse)
+    func dataRequest(with urlRequest: URLRequest, completion: @escaping NetworkSessionDataResponse)
+}
+
+extension URLSession: NetworkSession {
+    func dataRequest(with url: URL, completion: @escaping NetworkSessionDataResponse) {
+        let task = dataTask(with: url) { data, response, error in
+            completion(data, response, error)
+        }
+
+        task.resume()
+    }
+
+    func dataRequest(with urlRequest: URLRequest, completion: @escaping NetworkSessionDataResponse) {
+        let task = dataTask(with: urlRequest) { data, response, error in
+            completion(data, response, error)
+        }
+
+        task.resume()
+    }
+}

Client/Networking/NetworkSessionMock.swift

@@ -0,0 +1,19 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import Foundation
+
+class NetworkSessionMock: NetworkSession {
+    var data: Data?
+    var response: URLResponse?
+    var error: Error?
+
+    func dataRequest(with url: URL, completion: @escaping NetworkSessionDataResponse) {
+        completion(data, response, error)
+    }
+
+    func dataRequest(with urlRequest: URLRequest, completion: @escaping NetworkSessionDataResponse) {
+        completion(data, response, error)
+    }
+}