Fix #1442: Adblock rust integration. (#1441, cherry-pick #1469)

`beta` backport with abp-rust to `development`. This actual merge happening on PR #1469.

CHANGELOG.md

@@ -35,4 +35,4 @@
  - Fixed auto-focus of a tab when a link is opened using the share sheet. ([#698](https://github.com/brave/brave-ios/issues/698))
  - Fixed missing translations when viewing introduction summary under private tab. ([#1239](https://github.com/brave/brave-ios/issues/1239))
  - Fixed webcompat issues with https://borsen.dk/ due to Brave shields. ([#1061](https://github.com/brave/brave-ios/issues/1061))
-
+

Client.xcodeproj/project.pbxproj

@@ -74,6 +74,7 @@
 		0AA21E4A2302C4CC00358988 /* webauth_verify_key.json in Resources */ = {isa = PBXBuildFile; fileRef = 0AA21E472302C4CC00358988 /* webauth_verify_key.json */; };
 		0AA21E4B2302C4CC00358988 /* webauth_insert_key.json in Resources */ = {isa = PBXBuildFile; fileRef = 0AA21E482302C4CC00358988 /* webauth_insert_key.json */; };
 		0AA21E4C2302C4CC00358988 /* webauth_touch_key.json in Resources */ = {isa = PBXBuildFile; fileRef = 0AA21E492302C4CC00358988 /* webauth_touch_key.json */; };
+		0AA4AA99231849450009AAEF /* ABPFilterParserData.dat in Resources */ = {isa = PBXBuildFile; fileRef = 0AA4AA98231849440009AAEF /* ABPFilterParserData.dat */; };
 		0AA4FC392109D685000B173A /* CRUDProtocols.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AA4FC382109D685000B173A /* CRUDProtocols.swift */; };
 		0AAAAC972249174A009A8763 /* SyncAlerts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AAAAC962249174A009A8763 /* SyncAlerts.swift */; };
 		0AAAACAD22491CC7009A8763 /* ErrorExtensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AAAACAC22491CC7009A8763 /* ErrorExtensions.swift */; };
@@ -1220,6 +1221,7 @@
 		0AA21E472302C4CC00358988 /* webauth_verify_key.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = webauth_verify_key.json; sourceTree = "<group>"; };
 		0AA21E482302C4CC00358988 /* webauth_insert_key.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = webauth_insert_key.json; sourceTree = "<group>"; };
 		0AA21E492302C4CC00358988 /* webauth_touch_key.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = webauth_touch_key.json; sourceTree = "<group>"; };
+		0AA4AA98231849440009AAEF /* ABPFilterParserData.dat */ = {isa = PBXFileReference; lastKnownFileType = file; path = ABPFilterParserData.dat; sourceTree = "<group>"; };
 		0AA4FC382109D685000B173A /* CRUDProtocols.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CRUDProtocols.swift; sourceTree = "<group>"; };
 		0AAAAC962249174A009A8763 /* SyncAlerts.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncAlerts.swift; sourceTree = "<group>"; };
 		0AAAACAC22491CC7009A8763 /* ErrorExtensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ErrorExtensions.swift; sourceTree = "<group>"; };
@@ -2640,6 +2642,7 @@
 			children = (
 				0A063D4B22EB2945001CE50B /* ablock_rust_lib.h */,
 				0A063D4A22EB2945001CE50B /* libadblock.a */,
+				0AA4AA98231849440009AAEF /* ABPFilterParserData.dat */,
 			);
 			path = Resources;
 			sourceTree = "<group>";
@@ -5215,6 +5218,7 @@
 				D0FCF8081FE4772D004A7995 /* MainFrameAtDocumentStart.js in Resources */,
 				0A0D3D5021A5609600BEE65B /* SafeBrowsingError.html in Resources */,
 				74821FFE1DB6D3AC00EEEA72 /* MailSchemes.plist in Resources */,
+				0AA4AA99231849450009AAEF /* ABPFilterParserData.dat in Resources */,
 				4422D57621C05DE600BF1855 /* httpse.leveldb.tgz in Resources */,
 				FA9294011D6584A200AC8D33 /* QRCode.xcassets in Resources */,
 				D308EE561CBF0BF5006843F2 /* CertError.css in Resources */,

Client/Assets/Images.xcassets/WebAuth/enter_pin.imageset/Contents.json

@@ -20,4 +20,4 @@
     "version" : 1,
     "author" : "xcode"
   }
-}
+}

Client/WebFilters/AdblockResourceDownloader.swift

@@ -23,6 +23,8 @@ class AdblockResourceDownloader {
 
     static let folderName = "abp-data"
 
+    static let endpoint = "https://adblock-data.s3.brave.com/iOS13"
+
     init(networkManager: NetworkManager = NetworkManager(), locale: String? = Locale.current.languageCode) {
         if locale == nil {
             log.warning("No locale provided, using default one(\"en\")")
@@ -68,15 +70,17 @@ class AdblockResourceDownloader {
         let fileName = type.identifier
 
         let completedDownloads = type.associatedFiles.map { fileType -> Deferred<AdBlockNetworkResource> in
-            let fileExtension = "." + fileType.rawValue
+            let fileExtension = fileType.rawValue
             let etagExtension = fileExtension + ".etag"
 
-            guard let resourceName = type.resourceName(for: fileType),
-                let url = URL(string: fileType.endpoint + resourceName + fileExtension) else {
+            guard let resourceName = type.resourceName(for: fileType), var url = type.endpoint else {
                 return Deferred<AdBlockNetworkResource>()
             }
 
-            let etag = fileFromDocumentsAsString(fileName + etagExtension, inFolder: folderName)
+            url.appendPathComponent(resourceName)
+            url.appendPathExtension(fileExtension)
+
+            let etag = fileFromDocumentsAsString("\(fileName).\(etagExtension)", inFolder: folderName)
             let request = nm.downloadResource(with: url, resourceType: .cached(etag: etag))
                 .mapQueue(queue) { resource in
                     AdBlockNetworkResource(resource: resource, fileType: fileType, type: type)

Client/WebFilters/AdblockResourcesMappings.swift

@@ -7,8 +7,8 @@ import Foundation
 struct AdblockResourcesMappings {
     static func generalAdblockName(for fileType: FileType) -> String? {
         switch fileType {
-        case .dat: return "rs-ABPFilterParserData"
-        case .json: return "latest"
+        case .dat: return "combined-included-rs"
+        case .json: return "combined-included"
         default: return nil
         }
     }
@@ -25,41 +25,35 @@ enum ResourceLocale: String {
         var resourceId = ""
 
         switch self {
-        case .ar: resourceId = "9FCEECEC-52B4-4487-8E57-8781E82C91D0"
-        case .bg: resourceId = "FD176DD1-F9A0-4469-B43E-B1764893DD5C"
-        case .zh: resourceId = "11F62B02-9D1F-4263-A7F8-77D2B55D4594"
-        case .cs: resourceId = "7CCB6921-7FDA-4A9B-B70A-12DD0A8F08EA"
-        case .de: resourceId = "E71426E7-E898-401C-A195-177945415F38"
-        case .da: resourceId = "9EF6A21C-5014-4199-95A2-A82491274203"
-        case .et: resourceId = "0783DBFD-B5E0-4982-9B4A-711BDDB925B7"
-        case .fi: resourceId = "1C6D8556-3400-4358-B9AD-72689D7B2C46"
-        case .fr: resourceId = "9852EFC4-99E4-4F2D-A915-9C3196C7A1DE"
-        case .el: resourceId = "6C0F4C7F-969B-48A0-897A-14583015A587"
-        case .hu: resourceId = "EDEEE15A-6FA9-4FAC-8CA8-3565508EAAC3"
-        case .id: resourceId = "93123971-5AE6-47BA-93EA-BE1E4682E2B6"
-        case .hi: resourceId = "4C07DB6B-6377-4347-836D-68702CF1494A"
-        case .fa: resourceId = "C3C2F394-D7BB-4BC2-9793-E0F13B2B5971"
-        case .is: resourceId = "48796273-E783-431E-B864-44D3DCEA66DC"
-        case .he: resourceId = "85F65E06-D7DA-4144-B6A5-E1AA965D1E47"
-        case .it: resourceId = "AB1A661D-E946-4F29-B47F-CA3885F6A9F7"
-        case .ja: resourceId = "03F91310-9244-40FA-BCF6-DA31B832F34D"
-        case .ko: resourceId = "1E6CF01B-AFC4-47D2-AE59-3E32A1ED094F"
-        case .lt: resourceId = "4E8B1A63-DEBE-4B8B-AD78-3811C632B353"
-        case .lv: resourceId = "15B64333-BAF9-4B77-ADC8-935433CD6F4C"
-        case .nl: resourceId = "9D644676-4784-4982-B94D-C9AB19098D2A"
-        case .pl: resourceId = "BF9234EB-4CB7-4CED-9FCB-F1FD31B0666C"
-        case .ru, .uk, .be: resourceId = "80470EEC-970F-4F2C-BF6B-4810520C72E6"
-        case .es: resourceId = "AE657374-1851-4DC4-892B-9212B13B15A7"
-        case .sl: resourceId = "418D293D-72A8-4A28-8718-A1EE40A45AAF"
-        case .sv: resourceId = "7DC2AC80-5BBC-49B8-B473-A31A1145CAC1"
-        case .tr: resourceId = "1BE19EFD-9191-4560-878E-30ECA72B5B3C"
-        case .vi: resourceId = "6A0209AC-9869-4FD6-A9DF-039B4200D52C"
-        }
-
-        switch fileType {
-        case .dat: return "rs-" + resourceId
-        case .json: return resourceId + "-latest"
-        case .tgz: return ""
+        case .ar: return "9FCEECEC-52B4-4487-8E57-8781E82C91D0"
+        case .bg: return "FD176DD1-F9A0-4469-B43E-B1764893DD5C"
+        case .zh: return "11F62B02-9D1F-4263-A7F8-77D2B55D4594"
+        case .cs: return "7CCB6921-7FDA-4A9B-B70A-12DD0A8F08EA"
+        case .de: return "E71426E7-E898-401C-A195-177945415F38"
+        case .da: return "9EF6A21C-5014-4199-95A2-A82491274203"
+        case .et: return "0783DBFD-B5E0-4982-9B4A-711BDDB925B7"
+        case .fi: return "1C6D8556-3400-4358-B9AD-72689D7B2C46"
+        case .fr: return "9852EFC4-99E4-4F2D-A915-9C3196C7A1DE"
+        case .el: return "6C0F4C7F-969B-48A0-897A-14583015A587"
+        case .hu: return "EDEEE15A-6FA9-4FAC-8CA8-3565508EAAC3"
+        case .id: return "93123971-5AE6-47BA-93EA-BE1E4682E2B6"
+        case .hi: return "4C07DB6B-6377-4347-836D-68702CF1494A"
+        case .fa: return "C3C2F394-D7BB-4BC2-9793-E0F13B2B5971"
+        case .is: return "48796273-E783-431E-B864-44D3DCEA66DC"
+        case .he: return "85F65E06-D7DA-4144-B6A5-E1AA965D1E47"
+        case .it: return "AB1A661D-E946-4F29-B47F-CA3885F6A9F7"
+        case .ja: return "03F91310-9244-40FA-BCF6-DA31B832F34D"
+        case .ko: return "1E6CF01B-AFC4-47D2-AE59-3E32A1ED094F"
+        case .lt: return "4E8B1A63-DEBE-4B8B-AD78-3811C632B353"
+        case .lv: return "15B64333-BAF9-4B77-ADC8-935433CD6F4C"
+        case .nl: return "9D644676-4784-4982-B94D-C9AB19098D2A"
+        case .pl: return "BF9234EB-4CB7-4CED-9FCB-F1FD31B0666C"
+        case .ru, .uk, .be: return "80470EEC-970F-4F2C-BF6B-4810520C72E6"
+        case .es: return "AE657374-1851-4DC4-892B-9212B13B15A7"
+        case .sl: return "418D293D-72A8-4A28-8718-A1EE40A45AAF"
+        case .sv: return "7DC2AC80-5BBC-49B8-B473-A31A1145CAC1"
+        case .tr: return "1BE19EFD-9191-4560-878E-30ECA72B5B3C"
+        case .vi: return "6A0209AC-9869-4FD6-A9DF-039B4200D52C"
         }
     }
 }

Client/WebFilters/AdblockerType.swift

@@ -9,14 +9,6 @@ private let log = Logger.browserLogger
 
 enum FileType: String {
     case dat, json, tgz
-
-    var endpoint: String {
-        switch self {
-        case .dat: return "https://adblock-data.s3.brave.com/4/"
-        case .json: return "https://adblock-data.s3.brave.com/ios/"
-        case .tgz: return "" // Httpse resources are not supported yet.
-        }
-    }
 }
 
 enum AdblockerType {
@@ -57,6 +49,17 @@ enum AdblockerType {
         }
     }
 
+    /// A name under which given resource is stored on server.
+    var endpoint: URL? {
+        guard var url = URL(string: AdblockResourceDownloader.endpoint) else { return nil }
+
+        if case .regional = self {
+            url.appendPathComponent("regional")
+        }
+
+        return url
+    }
+
     var blockListName: BlocklistName? {
         switch self {
         case .general: return BlocklistName.ad

Client/WebFilters/ShieldStats/Adblock/AdBlockStats.swift

@@ -28,9 +28,25 @@ class AdBlockStats: LocalAdblockResourceProtocol {
     }
 
     func startLoading() {
+        parseBundledGeneralBlocklist()
         loadDownloadedDatFiles()
     }
 
+    private func parseBundledGeneralBlocklist() {
+        guard let path = Bundle.main.path(forResource: bundledGeneralBlocklist, ofType: "dat") else {
+            log.error("Can't find path for bundled general blocklist")
+            return
+        }
+        let fileUrl = URL(fileURLWithPath: path)
+
+        do {
+            let data = try Data(contentsOf: fileUrl)
+            generalAdblockEngine.set(data: data)
+        } catch {
+            log.error("Failed to parse bundled general blocklist: \(error)")
+        }
+    }
+
     private func loadDownloadedDatFiles() {
         let fm = FileManager.default
 
@@ -122,6 +138,9 @@ class AdBlockStats: LocalAdblockResourceProtocol {
 
         if engine.set(data: data) {
             log.debug("Adblock file with id: \(id) deserialized successfully")
+            // Clearing the cache or checked urls.
+            // The new list can bring blocked resource that were previously set as not-blocked.
+            fifoCacheOfUrlsChecked = FifoDict()
             completion.fill(())
         } else {
             log.error("Failed to deserialize adblock list with id: \(id)")