Revert "Fix #4335: Deprecate ours HTTPSE in favor of the one bundled …

…in WKWebview (#4433)"

This reverts commit a3051a0.

BraveShared/BraveStrings.swift

@@ -686,7 +686,7 @@ extension Strings {
     public static let blockAdsAndTracking = NSLocalizedString("BlockAdsAndTracking", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Block Cross-Site Trackers", comment: "")
     public static let blockAdsAndTrackingDescription = NSLocalizedString("BlockAdsAndTrackingDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Prevents ads, popups, and trackers from loading.", comment: "")
     public static let HTTPSEverywhere = NSLocalizedString("HTTPSEverywhere", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Upgrade Connections to HTTPS", comment: "")
-    public static let HTTPSEverywhereDescription = NSLocalizedString("HTTPSEverywhereDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Opens sites using secure HTTPS instead of HTTP when possible.", comment: "")
+    public static let HTTPSEverywhereDescription = NSLocalizedString("HTTPSEverywhereDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Opens some sites using a secure HTTPS connection instead of plain HTTP.", comment: "")
     public static let blockPhishingAndMalware = NSLocalizedString("BlockPhishingAndMalware", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Block Phishing and Malware", comment: "")
     public static let googleSafeBrowsing = NSLocalizedString("GoogleSafeBrowsing", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Block dangerous sites", comment: "")
     public static let googleSafeBrowsingUsingWebKitDescription = NSLocalizedString("GoogleSafeBrowsingUsingWebKitDescription", tableName: "BraveShared", bundle: Bundle.braveShared, value: "Sends obfuscated URLs of some pages you visit to the Google Safe Browsing service, when your security is at risk.", comment: "")

BraveShared/Shields/BraveShield.swift

@@ -8,6 +8,7 @@ import Foundation
 public enum BraveShield {
     case AllOff
     case AdblockAndTp
+    case HTTPSE
     case SafeBrowsing
     case FpProtection
     case NoScript
@@ -18,6 +19,8 @@ public enum BraveShield {
             return false
         case .AdblockAndTp:
             return Preferences.Shields.blockAdsAndTracking.value
+        case .HTTPSE:
+            return Preferences.Shields.httpsEverywhere.value
         case .SafeBrowsing:
             return Preferences.Shields.blockPhishingAndMalware.value
         case .FpProtection:

Client/Application/Delegates/AppDelegate.swift

@@ -87,14 +87,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
 
         // Setup Adblock Stats and HTTPSE Stats.
         AdBlockStats.shared.startLoading()
-
-        // TODO: Downgrade to 14.5 once api becomes available.
-        if #available(iOS 15, *) {
-            // do nothing, use Apple's https solution.
-        } else {
-            HttpsEverywhereStats.shared.startLoading()
-        }
-
+        HttpsEverywhereStats.shared.startLoading()
 
         // Setup Application Shortcuts
         updateShortcutItems(application)

Client/Frontend/Browser/BrowserViewController.swift

@@ -2926,19 +2926,17 @@ extension BrowserViewController: PreferencesObserver {
             updateApplicationShortcuts()
         case Preferences.General.alwaysRequestDesktopSite.key:
             tabManager.reset()
-            tabManager.reloadSelectedTab()
+            self.tabManager.reloadSelectedTab()
         case Preferences.General.enablePullToRefresh.key:
             tabManager.selectedTab?.updatePullToRefreshVisibility()
         case Preferences.Shields.blockAdsAndTracking.key,
+             Preferences.Shields.httpsEverywhere.key,
              Preferences.Shields.blockScripts.key,
              Preferences.Shields.blockPhishingAndMalware.key,
              Preferences.Shields.blockImages.key,
              Preferences.Shields.fingerprintingProtection.key,
              Preferences.Shields.useRegionAdBlock.key:
             tabManager.allTabs.forEach { $0.webView?.reload() }
-        case Preferences.Shields.httpsEverywhere.key:
-            tabManager.reset()
-            tabManager.reloadSelectedTab()
         case Preferences.Privacy.blockAllCookies.key,
              Preferences.Shields.googleSafeBrowsing.key:
             // All `block all cookies` toggle requires a hard reset of Webkit configuration.

Client/Frontend/Browser/BrowserViewController/BrowserViewController+WKNavigationDelegate.swift

@@ -242,19 +242,16 @@ extension BrowserViewController: WKNavigationDelegate {
 
             pendingRequests[url.absoluteString] = navigationAction.request
 
-            // TODO: Downgrade to 14.5 once api becomes available.
-            if #available(iOS 15, *) {
-                // do nothing, use Apple's https solution.
-            } else {
-                if Preferences.Shields.httpsEverywhere.value,
-                   url.scheme == "http",
-                    let urlHost = url.normalizedHost() {
+            if let urlHost = url.normalizedHost() {
+                if let mainDocumentURL = navigationAction.request.mainDocumentURL, url.scheme == "http" {
+                    let domainForShields = Domain.getOrCreate(forUrl: mainDocumentURL, persistent: !isPrivateBrowsing)
                     HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldupgrade in
                         DispatchQueue.main.async {
-                            if shouldupgrade {
+                            if domainForShields.isShieldExpected(.HTTPSE, considerAllShieldsOption: true) && shouldupgrade {
                                 self.pendingHTTPUpgrades[urlHost] = navigationAction.request
                             }
                         }
+
                     }
                 }
             }

Client/Frontend/Browser/Playlist/Managers & Cache/PlaylistCacheLoader.swift

@@ -653,16 +653,12 @@ extension PlaylistWebLoader: WKNavigationDelegate {
 
             pendingRequests[url.absoluteString] = navigationAction.request
 
-            // TODO: Downgrade to 14.5 once api becomes available.
-            if #available(iOS 15, *) {
-                // do nothing, use Apple's https solution.
-            } else {
-                if Preferences.Shields.httpsEverywhere.value,
-                   url.scheme == "http",
-                    let urlHost = url.normalizedHost() {
+            if let urlHost = url.normalizedHost() {
+                if let mainDocumentURL = navigationAction.request.mainDocumentURL, url.scheme == "http" {
+                    let domainForShields = Domain.getOrCreate(forUrl: mainDocumentURL, persistent: false)
                     HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldupgrade in
                         DispatchQueue.main.async {
-                            if shouldupgrade {
+                            if domainForShields.isShieldExpected(.HTTPSE, considerAllShieldsOption: true) && shouldupgrade {
                                 self.pendingHTTPUpgrades[urlHost] = navigationAction.request
                             }
                         }
@@ -682,6 +678,7 @@ extension PlaylistWebLoader: WKNavigationDelegate {
                 // Force adblocking on
                 domainForShields.shield_allOff = 1
                 domainForShields.shield_adblockAndTp = true
+                domainForShields.shield_httpse = true
 
                 let (on, off) = BlocklistName.blocklists(forDomain: domainForShields)
                 let controller = webView.configuration.userContentController

Client/Frontend/Browser/Tab.swift

@@ -238,11 +238,6 @@ class Tab: NSObject {
             configuration!.ignoresViewportScaleLimits = true
             configuration!.mediaTypesRequiringUserActionForPlayback = Preferences.General.mediaAutoPlays.value ? [] : .all
 
-            // TODO: Downgrade to 14.5 once api becomes available.
-            if #available(iOS 15.0, *) {
-                configuration!.upgradeKnownHostsToHTTPS = Preferences.Shields.httpsEverywhere.value
-            }
-
             if configuration!.urlSchemeHandler(forURLScheme: InternalURL.scheme) == nil {
                 configuration!.setURLSchemeHandler(InternalSchemeHandler(), forURLScheme: InternalURL.scheme)
             }

Client/Frontend/Shields/AdvancedShieldsView.swift

@@ -10,6 +10,7 @@ import BraveUI
 class AdvancedShieldsView: UIStackView {
     let siteTitle = HeaderTitleView()
     let adsTrackersControl = ToggleView(title: Strings.blockAdsAndTracking)
+    let httpsUpgradesControl = ToggleView(title: Strings.HTTPSEverywhere)
     let blockMalwareControl = ToggleView(title: Strings.blockPhishing)
     let blockScriptsControl = ToggleView(title: Strings.blockScripts)
     let fingerprintingControl = ToggleView(title: Strings.fingerprintingProtection)
@@ -26,6 +27,7 @@ class AdvancedShieldsView: UIStackView {
         let rows: [UIView] = [
             siteTitle,
             adsTrackersControl,
+            httpsUpgradesControl,
             blockMalwareControl,
             blockScriptsControl,
             fingerprintingControl,

Client/Frontend/Shields/ShieldsViewController.swift

@@ -208,6 +208,7 @@ class ShieldsViewController: UIViewController, PopoverContentComponent {
         (.AdblockAndTp, shieldsView.advancedShieldView.adsTrackersControl, Preferences.Shields.blockAdsAndTracking),
         (.SafeBrowsing, shieldsView.advancedShieldView.blockMalwareControl, Preferences.Shields.blockPhishingAndMalware),
         (.NoScript, shieldsView.advancedShieldView.blockScriptsControl, Preferences.Shields.blockScripts),
+        (.HTTPSE, shieldsView.advancedShieldView.httpsUpgradesControl, Preferences.Shields.httpsEverywhere),
         (.FpProtection, shieldsView.advancedShieldView.fingerprintingControl, Preferences.Shields.fingerprintingProtection),
     ]
 

Client/WebFilters/ContentBlocker/BlocklistName.swift

@@ -19,14 +19,7 @@ class BlocklistName: CustomStringConvertible, ContentBlocker {
 
     /// List of all bundled content blockers.
     /// Regional lists are downloaded on fly and not included here.
-    static var allLists: Set<BlocklistName> {
-        // TODO: Downgrade to 14.5 once api becomes available.
-        if #available(iOS 15, *) {
-            return [.ad, .tracker, .image]
-        } else {
-            return [.ad, .tracker, .https, .image]
-        }
-    }
+    static var allLists: Set<BlocklistName> { return [.ad, .tracker, .https, .image] }
 
     let filename: String
     var rule: WKContentRuleList?
@@ -79,6 +72,10 @@ class BlocklistName: CustomStringConvertible, ContentBlocker {
 
         // TODO #159: Setup image shield
 
+        if domain.isShieldExpected(.HTTPSE, considerAllShieldsOption: true) {
+            onList.insert(.https)
+        }
+
         var offList = allLists.subtracting(onList)
         // Make sure to consider the regional list since the user may disable it globally
         if let regionalBlocker = regionalBlocker, !onList.contains(regionalBlocker) {

Client/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift

@@ -91,17 +91,12 @@ class TPStatsBlocklistChecker {
                 return
             }
 
-            // TODO: Downgrade to 14.5 once api becomes available.
-            if #available(iOS 15, *) {
-                // do nothing
-            } else {
-                HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldUpgrade in
-                    DispatchQueue.main.async {
-                        if enabledLists.contains(.https) && shouldUpgrade {
-                            completion(BlocklistName.https)
-                        } else {
-                            completion(nil)
-                        }
+            HttpsEverywhereStats.shared.shouldUpgrade(url) { shouldUpgrade in
+                DispatchQueue.main.async {
+                    if enabledLists.contains(.https) && shouldUpgrade {
+                        completion(BlocklistName.https)
+                    } else {
+                        completion(nil)
                     }
                 }
             }

Data/models/Domain.swift

@@ -18,10 +18,7 @@ public final class Domain: NSManagedObject, CRUD {
 
     @NSManaged public var shield_allOff: NSNumber?
     @NSManaged public var shield_adblockAndTp: NSNumber?
-
-    @available(*, deprecated, message: "Per domain HTTPSE shield is currently unused.")
     @NSManaged public var shield_httpse: NSNumber?
-
     @NSManaged public var shield_noScript: NSNumber?
     @NSManaged public var shield_fpProtection: NSNumber?
     @NSManaged public var shield_safeBrowsing: NSNumber?
@@ -84,17 +81,19 @@ public final class Domain: NSManagedObject, CRUD {
                 return self.shield_allOff?.boolValue ?? false
             case .AdblockAndTp:
                 return self.shield_adblockAndTp?.boolValue ?? Preferences.Shields.blockAdsAndTracking.value
+            case .HTTPSE:
+                return self.shield_httpse?.boolValue ?? Preferences.Shields.httpsEverywhere.value
             case .SafeBrowsing:
                 return self.shield_safeBrowsing?.boolValue ?? Preferences.Shields.blockPhishingAndMalware.value
             case .FpProtection:
                 return self.shield_fpProtection?.boolValue ?? Preferences.Shields.fingerprintingProtection.value
             case .NoScript:
                 return self.shield_noScript?.boolValue ?? Preferences.Shields.blockScripts.value
             }
-        }()
+        }
 
         let isAllShieldsOff = Bool(truncating: shield_allOff ?? NSNumber(value: 0))
-        let isSpecificShieldOn = isShieldOn
+        let isSpecificShieldOn = isShieldOn()
         return considerAllShieldsOption ? !isAllShieldsOff && isSpecificShieldOn : isSpecificShieldOn
     }
 
@@ -121,6 +120,7 @@ public final class Domain: NSManagedObject, CRUD {
                     httpsDomain.shield_noScript = domain.shield_noScript
                     httpsDomain.shield_fpProtection = domain.shield_fpProtection
                     httpsDomain.shield_safeBrowsing = domain.shield_safeBrowsing
+                    httpsDomain.shield_httpse = domain.shield_httpse
                     // Could call `domain.delete()` here (or add to batch to delete)
                 }
             }
@@ -229,6 +229,12 @@ extension Domain {
         switch shield {
         case .AllOff: shield_allOff = setting
         case .AdblockAndTp: shield_adblockAndTp = setting
+        case .HTTPSE:
+          shield_httpse = setting
+
+          // HTTPSE must be scheme indepedent or user may get stuck not being able to access the http version
+          //  of a website (turning off httpse for an upgraded-https domain does not allow access to http version)
+          self.domainForInverseHttpScheme(context: context)?.shield_httpse = setting
         case .SafeBrowsing: shield_safeBrowsing = setting
         case .FpProtection: shield_fpProtection = setting
         case .NoScript: shield_noScript = setting
@@ -242,6 +248,8 @@ extension Domain {
             return self.shield_allOff?.boolValue
         case .AdblockAndTp:
             return self.shield_adblockAndTp?.boolValue
+        case .HTTPSE:
+            return self.shield_httpse?.boolValue
         case .SafeBrowsing:
             return self.shield_safeBrowsing?.boolValue
         case .FpProtection:

DataTests/DomainTests.swift

@@ -50,6 +50,7 @@ class DomainTests: CoreDataTestCase {
 
         let domain = Domain.getOrCreate(forUrl: url, persistent: true)
         XCTAssertTrue(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
+        XCTAssertTrue(domain.isShieldExpected(BraveShield.HTTPSE, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AllOff, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.NoScript, considerAllShieldsOption: true))
@@ -68,6 +69,7 @@ class DomainTests: CoreDataTestCase {
         }
 
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
+        XCTAssertFalse(domain.isShieldExpected(BraveShield.HTTPSE, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: false))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AllOff, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.NoScript, considerAllShieldsOption: true))
@@ -78,6 +80,7 @@ class DomainTests: CoreDataTestCase {
         }
 
         XCTAssertTrue(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
+        XCTAssertTrue(domain.isShieldExpected(BraveShield.HTTPSE, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.AllOff, considerAllShieldsOption: true))
         XCTAssertFalse(domain.isShieldExpected(BraveShield.NoScript, considerAllShieldsOption: true))
@@ -114,4 +117,38 @@ class DomainTests: CoreDataTestCase {
         XCTAssertTrue(domain.isShieldExpected(BraveShield.SafeBrowsing, considerAllShieldsOption: true))
         XCTAssertTrue(domain.isShieldExpected(BraveShield.AdblockAndTp, considerAllShieldsOption: true))
     }
+
+    /// Testing HTTPSE
+    /// if setting an HTTP scheme, that HTTPS is also set
+    func testHTTPSEforHTTPsetter() {
+        backgroundSaveAndWaitForExpectation {
+            Domain.setBraveShield(forUrl: url, shield: .HTTPSE, isOn: true, isPrivateBrowsing: false)
+        }
+
+        // Should be one for HTTP and one for HTTPS schemes
+        XCTAssertEqual(try! DataController.viewContext.count(for: fetchRequest), 2)
+
+        let domainRefetch1 = Domain.getOrCreate(forUrl: url, persistent: true)
+        XCTAssertEqual(domainRefetch1.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
+
+        let domainRefetch2 = Domain.getOrCreate(forUrl: urlHTTPS, persistent: true)
+        XCTAssertEqual(domainRefetch2.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
+    }
+
+    /// Testing HTTPSE
+    /// if setting an HTTPS scheme, that HTTP is also set
+    func testHTTPSEforHTTPSsetter() {
+        backgroundSaveAndWaitForExpectation {
+            Domain.setBraveShield(forUrl: url2HTTPS, shield: .HTTPSE, isOn: true, isPrivateBrowsing: false)
+        }
+
+        // Should be one for HTTP and one for HTTPS schemes
+        XCTAssertEqual(try! DataController.viewContext.count(for: fetchRequest), 2)
+
+        let domainRefetch1 = Domain.getOrCreate(forUrl: url2, persistent: true)
+        XCTAssertEqual(domainRefetch1.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
+
+        let domainRefetch2 = Domain.getOrCreate(forUrl: url2HTTPS, persistent: true)
+        XCTAssertEqual(domainRefetch2.isShieldExpected(.HTTPSE, considerAllShieldsOption: true), true)
+    }
 }