Fix #4335: Deprecate ours HTTPSE in favor of the one bundled in WKWeb…

…view (#4433)

BraveShared/BraveStrings.swift

@@ -686,7 +686,7 @@ extension Strings {
     public static let blockAdsAndTracking = NSLocalizedString("BlockAdsAndTracking", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Block Cross-Site Trackers", comment: "")
     public static let blockAdsAndTrackingDescription = NSLocalizedString("BlockAdsAndTrackingDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Prevents ads, popups, and trackers from loading.", comment: "")
     public static let HTTPSEverywhere = NSLocalizedString("HTTPSEverywhere", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Upgrade Connections to HTTPS", comment: "")
-    public static let HTTPSEverywhereDescription = NSLocalizedString("HTTPSEverywhereDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Opens some sites using a secure HTTPS connection instead of plain HTTP.", comment: "")
+    public static let HTTPSEverywhereDescription = NSLocalizedString("HTTPSEverywhereDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Opens sites using secure HTTPS instead of HTTP when possible.", comment: "")
     public static let blockPhishingAndMalware = NSLocalizedString("BlockPhishingAndMalware", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Block Phishing and Malware", comment: "")
     public static let googleSafeBrowsing = NSLocalizedString("GoogleSafeBrowsing", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Block dangerous sites", comment: "")
     public static let googleSafeBrowsingUsingWebKitDescription = NSLocalizedString("GoogleSafeBrowsingUsingWebKitDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Sends obfuscated URLs of some pages you visit to the Google Safe Browsing service, when your security is at risk.", comment: "")

BraveShared/Shields/BraveShield.swift

@@ -8,7 +8,6 @@ import Foundation
 public enum BraveShield {
     case AllOff
     case AdblockAndTp
-    case HTTPSE
     case SafeBrowsing
     case FpProtection
     case NoScript
@@ -19,8 +18,6 @@ public enum BraveShield {
             return false
         case .AdblockAndTp:
             return Preferences.Shields.blockAdsAndTracking.value
-        case .HTTPSE:
-            return Preferences.Shields.httpsEverywhere.value
         case .SafeBrowsing:
             return Preferences.Shields.blockPhishingAndMalware.value
         case .FpProtection:

Client/Application/Delegates/AppDelegate.swift

@@ -87,7 +87,14 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
 
         // Setup Adblock Stats and HTTPSE Stats.
         AdBlockStats.shared.startLoading()
-        HttpsEverywhereStats.shared.startLoading()
+
+        // TODO: Downgrade to 14.5 once api becomes available.
+        if #available(iOS 15, *) {
+            // do nothing, use Apple's https solution.
+        } else {
+            HttpsEverywhereStats.shared.startLoading()
+        }
+
 
         // Setup Application Shortcuts
         updateShortcutItems(application)

Client/Frontend/Browser/BrowserViewController.swift

@@ -2926,17 +2926,19 @@ extension BrowserViewController: PreferencesObserver {
             updateApplicationShortcuts()
         case Preferences.General.alwaysRequestDesktopSite.key:
             tabManager.reset()
-            self.tabManager.reloadSelectedTab()
+            tabManager.reloadSelectedTab()
         case Preferences.General.enablePullToRefresh.key:
             tabManager.selectedTab?.updatePullToRefreshVisibility()
         case Preferences.Shields.blockAdsAndTracking.key,
-             Preferences.Shields.httpsEverywhere.key,
              Preferences.Shields.blockScripts.key,
              Preferences.Shields.blockPhishingAndMalware.key,
              Preferences.Shields.blockImages.key,
              Preferences.Shields.fingerprintingProtection.key,
              Preferences.Shields.useRegionAdBlock.key:
             tabManager.allTabs.forEach { $0.webView?.reload() }
+        case Preferences.Shields.httpsEverywhere.key:
+            tabManager.reset()
+            tabManager.reloadSelectedTab()
         case Preferences.Privacy.blockAllCookies.key,
              Preferences.Shields.googleSafeBrowsing.key:
             // All `block all cookies` toggle requires a hard reset of Webkit configuration.

Client/Frontend/Browser/BrowserViewController/BrowserViewController+WKNavigationDelegate.swift

@@ -242,16 +242,19 @@ extension BrowserViewController: WKNavigationDelegate {
 
             pendingRequests[url.absoluteString] = navigationAction.request
 
-            if let urlHost = url.normalizedHost() {
-                if let mainDocumentURL = navigationAction.request.mainDocumentURL, url.scheme == "http" {
-                    let domainForShields = Domain.getOrCreate(forUrl: mainDocumentURL, persistent: !isPrivateBrowsing)
+            // TODO: Downgrade to 14.5 once api becomes available.
+            if #available(iOS 15, *) {
+                // do nothing, use Apple's https solution.
+            } else {
+                if Preferences.Shields.httpsEverywhere.value,
+                   url.scheme == "http",
+                    let urlHost = url.normalizedHost() {
                     HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldupgrade in
                         DispatchQueue.main.async {
-                            if domainForShields.isShieldExpected(.HTTPSE, considerAllShieldsOption: true) && shouldupgrade {
+                            if shouldupgrade {
                                 self.pendingHTTPUpgrades[urlHost] = navigationAction.request
                             }
                         }
-
                     }
                 }
             }

Client/Frontend/Browser/Playlist/Managers & Cache/PlaylistCacheLoader.swift

@@ -653,12 +653,16 @@ extension PlaylistWebLoader: WKNavigationDelegate {
 
             pendingRequests[url.absoluteString] = navigationAction.request
 
-            if let urlHost = url.normalizedHost() {
-                if let mainDocumentURL = navigationAction.request.mainDocumentURL, url.scheme == "http" {
-                    let domainForShields = Domain.getOrCreate(forUrl: mainDocumentURL, persistent: false)
+            // TODO: Downgrade to 14.5 once api becomes available.
+            if #available(iOS 15, *) {
+                // do nothing, use Apple's https solution.
+            } else {
+                if Preferences.Shields.httpsEverywhere.value,
+                   url.scheme == "http",
+                    let urlHost = url.normalizedHost() {
                     HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldupgrade in
                         DispatchQueue.main.async {
-                            if domainForShields.isShieldExpected(.HTTPSE, considerAllShieldsOption: true) && shouldupgrade {
+                            if shouldupgrade {
                                 self.pendingHTTPUpgrades[urlHost] = navigationAction.request
                             }
                         }
@@ -678,7 +682,6 @@ extension PlaylistWebLoader: WKNavigationDelegate {
                 // Force adblocking on
                 domainForShields.shield_allOff = 1
                 domainForShields.shield_adblockAndTp = true
-                domainForShields.shield_httpse = true
 
                 let (on, off) = BlocklistName.blocklists(forDomain: domainForShields)
                 let controller = webView.configuration.userContentController

Client/Frontend/Browser/Tab.swift

@@ -238,6 +238,11 @@ class Tab: NSObject {
             configuration!.ignoresViewportScaleLimits = true
             configuration!.mediaTypesRequiringUserActionForPlayback = Preferences.General.mediaAutoPlays.value ? [] : .all
 
+            // TODO: Downgrade to 14.5 once api becomes available.
+            if #available(iOS 15.0, *) {
+                configuration!.upgradeKnownHostsToHTTPS = Preferences.Shields.httpsEverywhere.value
+            }
+
             if configuration!.urlSchemeHandler(forURLScheme: InternalURL.scheme) == nil {
                 configuration!.setURLSchemeHandler(InternalSchemeHandler(), forURLScheme: InternalURL.scheme)
             }

Client/Frontend/Shields/AdvancedShieldsView.swift

@@ -10,7 +10,6 @@ import BraveUI
 class AdvancedShieldsView: UIStackView {
     let siteTitle = HeaderTitleView()
     let adsTrackersControl = ToggleView(title: Strings.blockAdsAndTracking)
-    let httpsUpgradesControl = ToggleView(title: Strings.HTTPSEverywhere)
     let blockMalwareControl = ToggleView(title: Strings.blockPhishing)
     let blockScriptsControl = ToggleView(title: Strings.blockScripts)
     let fingerprintingControl = ToggleView(title: Strings.fingerprintingProtection)
@@ -27,7 +26,6 @@ class AdvancedShieldsView: UIStackView {
         let rows: [UIView] = [
             siteTitle,
             adsTrackersControl,
-            httpsUpgradesControl,
             blockMalwareControl,
             blockScriptsControl,
             fingerprintingControl,

Client/Frontend/Shields/ShieldsViewController.swift

@@ -208,7 +208,6 @@ class ShieldsViewController: UIViewController, PopoverContentComponent {
         (.AdblockAndTp, shieldsView.advancedShieldView.adsTrackersControl, Preferences.Shields.blockAdsAndTracking),
         (.SafeBrowsing, shieldsView.advancedShieldView.blockMalwareControl, Preferences.Shields.blockPhishingAndMalware),
         (.NoScript, shieldsView.advancedShieldView.blockScriptsControl, Preferences.Shields.blockScripts),
-        (.HTTPSE, shieldsView.advancedShieldView.httpsUpgradesControl, Preferences.Shields.httpsEverywhere),
         (.FpProtection, shieldsView.advancedShieldView.fingerprintingControl, Preferences.Shields.fingerprintingProtection),
     ]
 

Client/WebFilters/ContentBlocker/BlocklistName.swift

@@ -19,7 +19,14 @@ class BlocklistName: CustomStringConvertible, ContentBlocker {
 
     /// List of all bundled content blockers.
     /// Regional lists are downloaded on fly and not included here.
-    static var allLists: Set<BlocklistName> { return [.ad, .tracker, .https, .image] }
+    static var allLists: Set<BlocklistName> {
+        // TODO: Downgrade to 14.5 once api becomes available.
+        if #available(iOS 15, *) {
+            return [.ad, .tracker, .image]
+        } else {
+            return [.ad, .tracker, .https, .image]
+        }
+    }
 
     let filename: String
     var rule: WKContentRuleList?
@@ -72,10 +79,6 @@ class BlocklistName: CustomStringConvertible, ContentBlocker {
 
         // TODO #159: Setup image shield
 
-        if domain.isShieldExpected(.HTTPSE, considerAllShieldsOption: true) {
-            onList.insert(.https)
-        }
-
         var offList = allLists.subtracting(onList)
         // Make sure to consider the regional list since the user may disable it globally
         if let regionalBlocker = regionalBlocker, !onList.contains(regionalBlocker) {

Client/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift

@@ -91,12 +91,17 @@ class TPStatsBlocklistChecker {
                 return
             }
 
-            HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldUpgrade in
-                DispatchQueue.main.async {
-                    if enabledLists.contains(.https) && shouldUpgrade {
-                        completion(BlocklistName.https)
-                    } else {
-                        completion(nil)
+            // TODO: Downgrade to 14.5 once api becomes available.
+            if #available(iOS 15, *) {
+                // do nothing
+            } else {
+                HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldUpgrade in
+                    DispatchQueue.main.async {
+                        if enabledLists.contains(.https) && shouldUpgrade {
+                            completion(BlocklistName.https)
+                        } else {
+                            completion(nil)
+                        }
                     }
                 }
             }

Data/models/Domain.swift

@@ -18,7 +18,10 @@ public final class Domain: NSManagedObject, CRUD {
 
     @NSManaged public var shield_allOff: NSNumber?
     @NSManaged public var shield_adblockAndTp: NSNumber?
+
+    @available(*, deprecated, message: "Per domain HTTPSE shield is currently unused.")
     @NSManaged public var shield_httpse: NSNumber?
+
     @NSManaged public var shield_noScript: NSNumber?
     @NSManaged public var shield_fpProtection: NSNumber?
     @NSManaged public var shield_safeBrowsing: NSNumber?
@@ -81,19 +84,17 @@ public final class Domain: NSManagedObject, CRUD {
                 return self.shield_allOff?.boolValue ?? false
             case .AdblockAndTp:
                 return self.shield_adblockAndTp?.boolValue ?? Preferences.Shields.blockAdsAndTracking.value
-            case .HTTPSE:
-                return self.shield_httpse?.boolValue ?? Preferences.Shields.httpsEverywhere.value
             case .SafeBrowsing:
                 return self.shield_safeBrowsing?.boolValue ?? Preferences.Shields.blockPhishingAndMalware.value
             case .FpProtection:
                 return self.shield_fpProtection?.boolValue ?? Preferences.Shields.fingerprintingProtection.value
             case .NoScript:
                 return self.shield_noScript?.boolValue ?? Preferences.Shields.blockScripts.value
             }
-        }
+        }()
 
         let isAllShieldsOff = Bool(truncating: shield_allOff ?? NSNumber(value: 0))
-        let isSpecificShieldOn = isShieldOn()
+        let isSpecificShieldOn = isShieldOn
         return considerAllShieldsOption ? !isAllShieldsOff && isSpecificShieldOn : isSpecificShieldOn
     }
 
@@ -120,7 +121,6 @@ public final class Domain: NSManagedObject, CRUD {
                     httpsDomain.shield_noScript = domain.shield_noScript
                     httpsDomain.shield_fpProtection = domain.shield_fpProtection
                     httpsDomain.shield_safeBrowsing = domain.shield_safeBrowsing
-                    httpsDomain.shield_httpse = domain.shield_httpse
                     // Could call `domain.delete()` here (or add to batch to delete)
                 }
             }
@@ -229,12 +229,6 @@ extension Domain {
         switch shield {
         case .AllOff: shield_allOff = setting
         case .AdblockAndTp: shield_adblockAndTp = setting
-        case .HTTPSE:
-          shield_httpse = setting
-
-          // HTTPSE must be scheme indepedent or user may get stuck not being able to access the http version
-          //  of a website (turning off httpse for an upgraded-https domain does not allow access to http version)
-          self.domainForInverseHttpScheme(context: context)?.shield_httpse = setting
         case .SafeBrowsing: shield_safeBrowsing = setting
         case .FpProtection: shield_fpProtection = setting
         case .NoScript: shield_noScript = setting
@@ -248,8 +242,6 @@ extension Domain {
             return self.shield_allOff?.boolValue
         case .AdblockAndTp:
             return self.shield_adblockAndTp?.boolValue
-        case .HTTPSE:
-            return self.shield_httpse?.boolValue
         case .SafeBrowsing:
             return self.shield_safeBrowsing?.boolValue
         case .FpProtection:

DataTests/DomainTests.swift

@@ -50,7 +50,6 @@ class DomainTests: CoreDataTestCase {
 
         let domain = Domain.getOrCreate(forUrl: url, persistent: true)
         XCTAssertTrue(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
-        XCTAssertTrue(domain.isShieldExpected(BraveShield.HTTPSE, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AllOff, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.NoScript, considerAllShieldsOption: true))
@@ -69,7 +68,6 @@ class DomainTests: CoreDataTestCase {
         }
 
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
-        XCTAssertFalse(domain.isShieldExpected(BraveShield.HTTPSE, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: false))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AllOff, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.NoScript, considerAllShieldsOption: true))
@@ -80,7 +78,6 @@ class DomainTests: CoreDataTestCase {
         }
 
         XCTAssertTrue(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
-        XCTAssertTrue(domain.isShieldExpected(BraveShield.HTTPSE, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AllOff, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.NoScript, considerAllShieldsOption: true))
@@ -117,38 +114,4 @@ class DomainTests: CoreDataTestCase {
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
     }
-
-    /// Testing HTTPSE
-    /// if setting an HTTP scheme, that HTTPS is also set
-    func testHTTPSEforHTTPsetter() {
-        backgroundSaveAndWaitForExpectation {
-            Domain.setBraveShield(forUrl: url, shield: .HTTPSE, isOn: true, isPrivateBrowsing: false)
-        }
-
-        // Should be one for HTTP and one for HTTPS schemes
-        XCTAssertEqual(try! DataController.viewContext.count(for: fetchRequest), 2)
-
-        let domainRefetch1 = Domain.getOrCreate(forUrl: url, persistent: true)
-        XCTAssertEqual(domainRefetch1.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
-
-        let domainRefetch2 = Domain.getOrCreate(forUrl: urlHTTPS, persistent: true)
-        XCTAssertEqual(domainRefetch2.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
-    }
-
-    /// Testing HTTPSE
-    /// if setting an HTTPS scheme, that HTTP is also set
-    func testHTTPSEforHTTPSsetter() {
-        backgroundSaveAndWaitForExpectation {
-            Domain.setBraveShield(forUrl: url2HTTPS, shield: .HTTPSE, isOn: true, isPrivateBrowsing: false)
-        }
-
-        // Should be one for HTTP and one for HTTPS schemes
-        XCTAssertEqual(try! DataController.viewContext.count(for: fetchRequest), 2)
-
-        let domainRefetch1 = Domain.getOrCreate(forUrl: url2, persistent: true)
-        XCTAssertEqual(domainRefetch1.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
-
-        let domainRefetch2 = Domain.getOrCreate(forUrl: url2HTTPS, persistent: true)
-        XCTAssertEqual(domainRefetch2.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
-    }
 }