Fix #1918: Send in page innerText to ads classifier (#1930)

Instead of sending plain HTML, we now send in the document's `innerText`, which doesn't include any HTML tags, images, etc. This allows the ads classifier to properly classify pages to their correct type
brave · Nov 13, 2019 · a6da496 · a6da496
1 parent 8346b2c
commit a6da496
Show file tree

Hide file tree

Showing 9 changed files with 60 additions and 14 deletions.
diff --git a/BraveRewards/BraveRewards.framework/BraveRewards b/BraveRewards/BraveRewards.framework/BraveRewards
diff --git a/BraveRewards/BraveRewards.framework/Headers/BATBraveAds.h b/BraveRewards/BraveRewards.framework/Headers/BATBraveAds.h
@@ -103,8 +103,9 @@ NS_SWIFT_NAME(BraveAds)
 
 #pragma mark - Reporting
 
-/// Report that a page has loaded in the current browser tab, and the HTML is available for analysis
-- (void)reportLoadedPageWithURL:(NSURL *)url html:(NSString *)html;
+/// Report that a page has loaded in the current browser tab, and the inner text
+/// within the page loaded for classification
+- (void)reportLoadedPageWithURL:(NSURL *)url innerText:(NSString *)text;
 
 /// Report that media has started on a tab with a given id
 - (void)reportMediaStartedWithTabId:(NSInteger)tabId NS_SWIFT_NAME(reportMediaStarted(tabId:));

diff --git a/BraveRewards/BraveRewards.framework/Headers/BATBraveLedger.h b/BraveRewards/BraveRewards.framework/Headers/BATBraveLedger.h
@@ -162,7 +162,7 @@ NS_SWIFT_NAME(BraveLedger)
 - (void)listOneTimeTips:(void (NS_NOESCAPE ^)(NSArray<BATPublisherInfo *> *))completion;
 
 - (void)tipPublisherDirectly:(BATPublisherInfo *)publisher
-                      amount:(int)amount
+                      amount:(double)amount
                     currency:(NSString *)currency
                   completion:(void (^)(BATResult result))completion;
 

diff --git a/BraveRewards/BraveRewards.framework/Headers/BATBraveRewards.h b/BraveRewards/BraveRewards.framework/Headers/BATBraveRewards.h
@@ -93,14 +93,14 @@ NS_SWIFT_NAME(BraveRewards)
                isPrivate:(BOOL)isPrivate;
 /// Report that a page has loaded in the current browser tab, and the HTML is available for analysis
 ///
-/// @note Send false for `shouldClassifyForAds` if the load happened due to tabs restoring
+/// @note Send nil for `adsInnerText` if the load happened due to tabs restoring
 ///       after app launch or if response header for the page load contains
 ///       "cache-control: no-store"
 - (void)reportLoadedPageWithURL:(NSURL *)url
                      faviconURL:(nullable NSURL *)faviconURL
                           tabId:(UInt32)tabId
                            html:(NSString *)html
-           shouldClassifyForAds:(BOOL)shouldClassify NS_SWIFT_NAME(reportLoadedPage(url:faviconUrl:tabId:html:shouldClassifyForAds:));
+                   adsInnerText:(nullable NSString *)adsInnerText NS_SWIFT_NAME(reportLoadedPage(url:faviconUrl:tabId:html:adsInnerText:));
 /// Report any XHR load happening in the page
 - (void)reportXHRLoad:(NSURL *)url
                 tabId:(UInt32)tabId

diff --git a/BraveRewardsUI/README.md b/BraveRewardsUI/README.md
@@ -6,5 +6,5 @@ The latest BraveRewards.framework was built on:
 
 ```
 brave-browser/c9404a71bb301d1303df1fcd2c24f7f614174fe9
-brave-core/888b808f99981e008bbac6c58123440eba64f230
+brave-core/be26e03107bebecb17e120f81998915d28129de2
 ```
diff --git a/BraveRewardsUI/Tipping/TippingViewController.swift b/BraveRewardsUI/Tipping/TippingViewController.swift
@@ -175,7 +175,7 @@ class TippingViewController: UIViewController, UIViewControllerTransitioningDele
           // TODO: Handle started tip process
         }
       } else {
-        self.state.ledger.tipPublisherDirectly(self.publisherInfo, amount: Int32(amount), currency: "BAT") { _ in
+        self.state.ledger.tipPublisherDirectly(self.publisherInfo, amount: Double(amount), currency: "BAT") { _ in
           // TODO: Handle started tip process
         }
       }

diff --git a/BraveShared/Extensions/StringExtensions.swift b/BraveShared/Extensions/StringExtensions.swift
@@ -14,4 +14,18 @@ extension String {
         }
         return nil
     }
+
+    /// Obtain a list of words in a given string
+    public var words: [String] {
+        var words: [String] = []
+        enumerateSubstrings(
+            in: startIndex..<endIndex,
+            options: .byWords
+        ) { (word, _, _, _) in
+            if let word = word {
+                words.append(word)
+            }
+        }
+        return words
+    }
 }
diff --git a/BraveSharedTests/StringExtensionTests.swift b/BraveSharedTests/StringExtensionTests.swift
@@ -27,4 +27,19 @@ class StringExtensionTests: XCTestCase {
         let schemelessURL = "brave.com"
         XCTAssertNotNil(schemelessURL.firstURL)
     }
+
+    func testWords() {
+        let longMultilinedText = """
+        Multiple words
+
+        On multiple lines.
+        
+        That will get stripped!\r
+        """
+
+        XCTAssertEqual(longMultilinedText.words, ["Multiple", "words", "On", "multiple", "lines", "That", "will", "get", "stripped"])
+
+        let wordsWithPunctuation = "\"It's a wonderful life—isn't it…\""
+        XCTAssertEqual(wordsWithPunctuation.words, ["It's", "a", "wonderful", "life", "isn't", "it"])
+    }
 }
diff --git a/Client/Frontend/Browser/BrowserViewController/BVC+Rewards.swift b/Client/Frontend/Browser/BrowserViewController/BVC+Rewards.swift
@@ -115,18 +115,34 @@ extension Tab {
     func reportPageLoad(to rewards: BraveRewards) {
         guard let webView = webView, let url = webView.url else { return }
         if url.isLocal || PrivateBrowsingManager.shared.isPrivateBrowsing { return }
+
+        var htmlBlob: String?
+        var classifierText: String?
 
-        let getHtmlToStringJSCall = "document.documentElement.outerHTML.toString()"
-        // Copy to var, as `shouldClassifyLoadsForAds` can be reset before JS completes
-        let shouldClassify = shouldClassifyLoadsForAds
-        webView.evaluateJavaScript(getHtmlToStringJSCall, completionHandler: { html, _ in
-            guard let htmlString = html as? String else { return }
+        let group = DispatchGroup()
+        group.enter()
+        webView.evaluateJavaScript("document.documentElement.outerHTML.toString()", completionHandler: { html, _ in
+            htmlBlob = html as? String
+            group.leave()
+        })
+
+        if shouldClassifyLoadsForAds {
+            group.enter()
+            webView.evaluateJavaScript("document.body.innerText", completionHandler: { text, _ in
+                // Get the list of words in the page and join them together with a space
+                // to send to the classifier
+                classifierText = (text as? String)?.words.joined(separator: " ")
+                group.leave()
+            })
+        }
+
+        group.notify(queue: .main) {
             let faviconURL = URL(string: self.displayFavicon?.url ?? "")
             if faviconURL == nil {
                 log.warning("No favicon found in \(self) to report to rewards panel")
             }
-            rewards.reportLoadedPage(url: url, faviconUrl: faviconURL, tabId: self.rewardsId, html: htmlString, shouldClassifyForAds: shouldClassify)
-        })
+            rewards.reportLoadedPage(url: url, faviconUrl: faviconURL, tabId: self.rewardsId, html: htmlBlob ?? "", adsInnerText: classifierText)
+        }
     }
 
     func reportPageNaviagtion(to rewards: BraveRewards) {