Include web discovery bundles in the brave extension (uplift to 1.31.x)

DEPS

@@ -14,6 +14,7 @@ deps = {
   "vendor/bat-native-tweetnacl": "https://github.com/brave-intl/bat-native-tweetnacl.git@800f9d40b7409239ff192e0be634764e747c7a75",
   "vendor/challenge_bypass_ristretto_ffi": "https://github.com/brave-intl/challenge-bypass-ristretto-ffi.git@f2eff7aca4ea04564e3647b93eb72f33ebdbf683",
   "vendor/gn-project-generators": "https://github.com/brave/gn-project-generators.git@b76e14b162aa0ce40f11920ec94bfc12da29e5d0",
+  "vendor/web-discovery-project": "https://github.com/brave/web-discovery-project@17236e350dbee79cefe79f281a344170794cadf7",
   "third_party/ethash/src": "https://github.com/chfast/ethash.git@e4a15c3d76dc09392c7efd3e30d84ee3b871e9ce",
   "third_party/bitcoin-core/src": "https://github.com/bitcoin/bitcoin.git@95ea54ba089610019a74c1176a2c7c0dba144b1c",
 }
@@ -45,6 +46,13 @@ hooks = [
     'condition': 'not checkout_android and not checkout_ios',
     'action': ['vpython3', 'script/download_rust_deps.py'],
   },
+  {
+    # Install Web Discovery Project dependencies for Windows, Linux, and macOS
+    'name': 'build_web_discovery_project',
+    'pattern': '.',
+    'condition': 'not checkout_android and not checkout_ios',
+    'action': ['vpython3', 'script/web_discovery_project.py', '--install', '--build'],
+  },
   {
     'name': 'hermetic_xcode',
     'pattern': '.',

app/brave_generated_resources.grd

@@ -1195,6 +1195,14 @@ By installing this extension, you are agreeing to the Google Widevine Terms of U
         Brave Ad Notification
       </message>
 
+      <!-- Brave Search -->
+      <message name="IDS_SETTINGS_WEB_DISCOVERY_LABEL" desc="Description that shows settings for toggle Web Discovery">
+        Web Discovery Project
+      </message>
+      <message name="IDS_SETTINGS_WEB_DISCOVERY_SUBLABEL" desc="Description that shows under main toggle setting for Web Discovery">
+        Contribute some anonymous search &amp; browsing data to refine Brave Search.
+      </message>
+
       <!-- Brave Tooltips -->
       <message name="IDS_BRAVE_TOOLTIPS_BRAVE_TOOLTIP_ACCESSIBLE_NAME" desc="Text to be spoken when a tooltip is shown">
         Brave Tooltip
@@ -1295,6 +1303,7 @@ By installing this extension, you are agreeing to the Google Widevine Terms of U
       <message name="IDS_BRAVE_P3A_ENABLE_SETTING_SUBITEM" desc="Description that shows under main toggle setting for P3A">
         This completely anonymised info helps Brave estimate the overall usage of certain features and make them better for you.
       </message>
+
       <!-- Web3 detection -->
       <message name="IDS_BRAVE_CRYPTO_WALLETS_LAZY_LOAD_TEXT" desc="Text that shows on the info bar when a Dapp is detected">
         Would you like to start Crypto Wallets for Ethereum support?

browser/brave_profile_prefs.cc

@@ -371,6 +371,11 @@ void RegisterProfilePrefs(user_prefs::PrefRegistrySyncable* registry) {
       prefs::kBraveDefaultSearchVersion,
       TemplateURLPrepopulateData::kBraveCurrentDataVersion);
 
+#if BUILDFLAG(ENABLE_EXTENSIONS)
+  // Web discovery extension, default false
+  registry->RegisterBooleanPref(kWebDiscoveryEnabled, false);
+#endif
+
 #if BUILDFLAG(ENABLE_SPEEDREADER)
   speedreader::SpeedreaderService::RegisterProfilePrefs(registry);
 #endif

browser/extensions/api/settings_private/brave_prefs_util.cc

@@ -27,6 +27,7 @@
 #include "components/gcm_driver/gcm_buildflags.h"
 #include "components/omnibox/browser/omnibox_prefs.h"
 #include "components/search_engines/search_engines_pref_names.h"
+#include "extensions/buildflags/buildflags.h"
 
 #if BUILDFLAG(ENABLE_BRAVE_WAYBACK_MACHINE)
 #include "brave/components/brave_wayback_machine/pref_names.h"
@@ -173,6 +174,11 @@ const PrefsUtil::TypedPrefMap& BravePrefsUtil::GetAllowlistedKeys() {
 #if BUILDFLAG(ENABLE_FTX)
   (*s_brave_allowlist)[kFTXNewTabPageShowFTX] =
       settings_api::PrefType::PREF_TYPE_BOOLEAN;
+#endif
+#if BUILDFLAG(ENABLE_EXTENSIONS)
+  // Web discovery prefs
+  (*s_brave_allowlist)[kWebDiscoveryEnabled] =
+      settings_api::PrefType::PREF_TYPE_BOOLEAN;
 #endif
   // Brave today prefs
   (*s_brave_allowlist)[kBraveTodayOptedIn] =

browser/resources/settings/BUILD.gn

@@ -9,6 +9,7 @@ import("//brave/components/brave_wayback_machine/buildflags/buildflags.gni")
 import("//brave/components/tor/buildflags/buildflags.gni")
 import("//chrome/browser/resources/tools/optimize_webui.gni")
 import("//chrome/common/features.gni")
+import("//extensions/buildflags/buildflags.gni")
 import("//third_party/closure_compiler/compile_js.gni")
 import("//tools/grit/grit_rule.gni")
 import("//tools/grit/preprocess_if_expr.gni")
@@ -82,6 +83,7 @@ generate_grd("build_grd") {
 }
 
 preprocess_if_expr("preprocess") {
+  defines = [ "enable_extensions=$enable_extensions" ]
   in_folder = "./"
   out_folder =
       "$root_gen_dir/chrome/browser/resources/settings/$preprocess_folder"
@@ -116,6 +118,7 @@ preprocess_if_expr("preprocess") {
     "brave_overrides/settings_ui.js",
     "brave_overrides/site_details.js",
     "brave_overrides/site_settings_page.js",
+    "brave_overrides/search_page.js",
     "brave_overrides/sync_account_control.js",
     "brave_overrides/sync_controls.js",
     "brave_reset_page/brave_reset_profile_dialog_behavior.js",

browser/resources/settings/brave_overrides/index.js

@@ -39,3 +39,4 @@ import './site_details.js'
 import './site_settings_page.js'
 import './sync_account_control.js'
 import './sync_controls.js'
+import './search_page.js'

browser/resources/settings/brave_overrides/search_page.js

@@ -0,0 +1,24 @@
+// Copyright (c) 2021 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// you can obtain one at http://mozilla.org/MPL/2.0/.
+
+import {I18nBehavior} from 'chrome://resources/js/i18n_behavior.m.js'
+import {RegisterPolymerTemplateModifications} from 'chrome://brave-resources/polymer_overriding.js'
+
+<if expr="enable_extensions">
+  RegisterPolymerTemplateModifications({
+    'settings-search-page': (templateContent) => {
+      const searchExplanation = templateContent.getElementById('searchExplanation')
+      searchExplanation.parentNode.insertAdjacentHTML('afterend', `
+        <settings-toggle-button
+          class="hr"
+          pref="{{prefs.brave.web_discovery_enabled}}"
+          label="${I18nBehavior.i18n('braveWebDiscoveryLabel')}"
+          sub-label="${I18nBehavior.i18n('braveWebDiscoverySubLabel')}"
+          learn-more-url="${I18nBehavior.i18n('webDiscoveryLearnMoreURL')}"
+        </settings-toggle-button>
+      `)
+    }
+  })
+</if>

browser/ui/BUILD.gn

@@ -256,6 +256,7 @@ source_set("ui") {
     "//brave/components/brave_rewards/resources:internals_generated_resources",
     "//brave/components/brave_rewards/resources:page_generated_resources",
     "//brave/components/brave_rewards/resources:tip_generated_resources",
+    "//brave/components/brave_search/common",
     "//brave/components/brave_shields/browser",
     "//brave/components/brave_vpn/buildflags",
     "//brave/components/brave_wallet/common/buildflags",

chromium_src/chrome/browser/ui/webui/settings/settings_localized_strings_provider.cc

@@ -18,6 +18,7 @@
 #include "chrome/common/pref_names.h"
 #include "components/grit/brave_components_strings.h"
 #include "components/prefs/pref_service.h"
+#include "extensions/buildflags/buildflags.h"
 
 namespace settings {
 void BraveAddLocalizedStrings(content::WebUIDataSource*, Profile*);
@@ -112,6 +113,11 @@ void BraveAddCommonStrings(content::WebUIDataSource* html_source,
 #if BUILDFLAG(ENABLE_BRAVE_VPN)
     {"showBraveVPNButton", IDS_SETTINGS_SHOW_VPN_BUTTON},
     {"showBraveVPNButtonSubLabel", IDS_SETTINGS_SHOW_VPN_BUTTON_SUB_LABEL},
+#endif
+  // Search settings
+#if BUILDFLAG(ENABLE_EXTENSIONS)
+    {"braveWebDiscoveryLabel", IDS_SETTINGS_WEB_DISCOVERY_LABEL},
+    {"braveWebDiscoverySubLabel", IDS_SETTINGS_WEB_DISCOVERY_SUBLABEL},
 #endif
     {"mruCyclingSettingLabel", IDS_SETTINGS_BRAVE_MRU_CYCLING_LABEL},
     {"speedreaderSettingLabel", IDS_SETTINGS_SPEEDREADER_LABEL},
@@ -326,6 +332,9 @@ void BraveAddCommonStrings(content::WebUIDataSource* html_source,
   html_source->AddString("webRTCLearnMoreURL", kWebRTCLearnMoreURL);
   html_source->AddString("googleLoginLearnMoreURL", kGoogleLoginLearnMoreURL);
   html_source->AddString("ipfsDNSLinkLearnMoreURL", kDNSLinkLearnMoreURL);
+#if BUILDFLAG(ENABLE_EXTENSIONS)
+  html_source->AddString("webDiscoveryLearnMoreURL", kWebDiscoveryLearnMoreUrl);
+#endif
   html_source->AddString("speedreaderLearnMoreURL", kSpeedreaderLearnMoreUrl);
   html_source->AddString(
       "getMoreExtensionsUrl",

common/BUILD.gn

@@ -70,6 +70,7 @@ source_set("pref_names") {
   deps = [
     "//brave/components/brave_vpn/buildflags",
     "//components/gcm_driver:gcm_buildflags",
+    "//extensions/buildflags:buildflags",
   ]
 }
 

common/DEPS

@@ -7,6 +7,7 @@ include_rules = [
   "+components/grit/brave_components_resources.h",
   "+components/grit/components_resources.h",
   "+components/version_info",
+  "+extensions/buildflags",
   "+extensions/common/url_pattern.h",
   "+third_party/widevine/cdm/buildflags.h",
 ]

common/pref_names.cc

@@ -87,6 +87,11 @@ const char kOtherBookmarksMigrated[] = "brave.other_bookmarks_migrated";
 const char kBraveShieldsSettingsVersion[] = "brave.shields_settings_version";
 const char kDefaultBrowserPromptEnabled[] =
     "brave.default_browser_prompt_enabled";
+
+#if BUILDFLAG(ENABLE_EXTENSIONS)
+const char kWebDiscoveryEnabled[] = "brave.web_discovery_enabled";
+#endif
+
 #if !BUILDFLAG(USE_GCM_FROM_PLATFORM)
 const char kBraveGCMChannelStatus[] = "brave.gcm.channel_status";
 #endif

common/pref_names.h

@@ -9,6 +9,7 @@
 #include "brave/components/brave_vpn/buildflags/buildflags.h"
 #include "build/build_config.h"
 #include "components/gcm_driver/gcm_buildflags.h"
+#include "extensions/buildflags/buildflags.h"
 
 extern const char kAdsBlocked[];
 extern const char kTrackersBlocked[];
@@ -74,6 +75,12 @@ extern const char kBraveShieldsSettingsVersion[];
 extern const char kBinanceAccessToken[];
 extern const char kBinanceRefreshToken[];
 extern const char kDefaultBrowserPromptEnabled[];
+
+#if BUILDFLAG(ENABLE_EXTENSIONS)
+// Web discovery project
+extern const char kWebDiscoveryEnabled[];
+#endif
+
 #if !BUILDFLAG(USE_GCM_FROM_PLATFORM)
 extern const char kBraveGCMChannelStatus[];
 #endif

common/url_constants.cc

@@ -25,3 +25,5 @@ const char kPermissionPromptLearnMoreUrl[] =
 const char kSpeedreaderLearnMoreUrl[] =
     "https://support.brave.com/hc/en-us/articles/"
     "360045031392-What-is-SpeedReader";
+const char kWebDiscoveryLearnMoreUrl[] =
+    "https://brave.com/browser/privacy/#web-discovery-project";

common/url_constants.h

@@ -22,6 +22,7 @@ extern const char kImportDataHelpURL[];
 extern const char kCryptoWalletsLearnMoreURL[];
 extern const char kPermissionPromptLearnMoreUrl[];
 extern const char kSpeedreaderLearnMoreUrl[];
+extern const char kWebDiscoveryLearnMoreUrl[];
 
 // This is introduced to replace |kDownloadChromeUrl| in
 // outdated_upgrade_bubble_view.cc"

components/brave_extension/extension/brave_extension/background.ts

@@ -6,6 +6,7 @@ import * as bluebird from 'bluebird'
 
 import './background/greaselion'
 import './background/today'
+import './background/webDiscoveryProject'
 
 // workaround https://github.com/DefinitelyTyped/DefinitelyTyped/issues/42084
 global.Promise = bluebird as any

components/brave_extension/extension/brave_extension/background/webDiscoveryProject.ts

@@ -0,0 +1,74 @@
+// Copyright (c) 2021 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// you can obtain one at http://mozilla.org/MPL/2.0/.
+
+import { App } from '../../../../../vendor/web-discovery-project/build'
+
+declare var window: any
+
+function onCommitted (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) {
+  // Only inject if page is acceptable protocol (to skip internal pages like
+  // devtools://, brave://, etc.). We also make sure to only inject if we
+  // receive the onCommitted for the main_frame (frameId === 0).
+  if (
+    details.frameId === 0 &&
+    (details.url.startsWith('https://') || details.url.startsWith('http://'))
+  ) {
+    chrome.tabs.executeScript(details.tabId, {
+      file: 'bundles/web-discovery-content-script.bundle.js',
+      matchAboutBlank: false,
+      runAt: 'document_end'
+    }, () => {
+      if (chrome.runtime.lastError) {
+        console.warn('[web-discovery-project] Could not inject script:', chrome.runtime.lastError)
+      }
+    })
+  }
+}
+
+if (App !== undefined) {
+  const APP = new App({
+    version: chrome.runtime.getManifest().version
+  })
+  window.WDP = APP
+
+  const WEB_DISCOVERY_PREF_KEY = 'brave.web_discovery_enabled'
+
+  const toggleWebDiscovery = (pref?: chrome.settingsPrivate.PrefObject) => {
+    if (pref && pref.type === chrome.settingsPrivate.PrefType.BOOLEAN) {
+      const enable = pref.value
+      if (enable) {
+        // enable
+        APP.start()
+          .then(
+            () => {
+              // Dynamically inject WDP content script so that users with the pref disabled
+              // don't need to pay the loading cost on each page.
+              chrome.webNavigation.onCommitted.addListener(onCommitted)
+            },
+            (err) => { console.error('[web-discovery]', err) }
+          )
+      } else {
+        // Stop injecting dynamic content scripts
+        if (chrome.webNavigation.onCommitted.hasListener(onCommitted)) {
+          chrome.webNavigation.onCommitted.removeListener(onCommitted)
+        }
+
+        // disable
+        if (APP.isRunning) {
+          APP.stop()
+        }
+      }
+    }
+  }
+
+  chrome.settingsPrivate.onPrefsChanged.addListener((prefs: chrome.settingsPrivate.PrefObject[]) => {
+    const pref = prefs.find(p => p.key === WEB_DISCOVERY_PREF_KEY)
+    toggleWebDiscovery(pref)
+  })
+
+  chrome.settingsPrivate.getPref(WEB_DISCOVERY_PREF_KEY, (pref: chrome.settingsPrivate.PrefObject) => {
+    toggleWebDiscovery(pref)
+  })
+}

components/brave_extension/extension/brave_extension/manifest.json

@@ -21,7 +21,9 @@
     "elementPicker.js"
   ],
   "background": {
-    "scripts": ["out/brave_extension_background.bundle.js"],
+    "scripts": [
+      "out/brave_extension_background.bundle.js"
+    ],
     "persistent": true
   },
   "externally_connectable": {
@@ -56,6 +58,7 @@
     }
   ],
   "permissions": [
+    "activeTab",
     "alarms",
     "contentSettings",
     "contextMenus",
@@ -69,9 +72,11 @@
     "webRequest",
     "*://*/*",
     "chrome://favicon/*",
+    "webRequestBlocking",
+    "unlimitedStorage",
     "<all_urls>"
   ],
-  "content_security_policy": "default-src 'self'; font-src 'self' data:; script-src 'self'; style-src 'unsafe-inline'; img-src 'self' data: chrome://favicon/; connect-src https://pcdn.brave.com https://brave-today-cdn.brave.com https://pcdn.bravesoftware.com https://brave-today-cdn.bravesoftware.com https://pcdn.brave.software https://brave-today-cdn.brave.software https://api.twitter.com 'self';",
+  "content_security_policy": "default-src 'self'; font-src 'self' data:; script-src 'self' 'wasm-eval'; style-src 'unsafe-inline'; img-src 'self' data: chrome://favicon/; connect-src * data: blob: filesystem:;",
   "incognito": "split",
   "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAupOLMy5Fd4dCSOtjcApsAQOnuBdTs+OvBVt/3P93noIrf068x0xXkvxbn+fpigcqfNamiJ5CjGyfx9zAIs7zcHwbxjOw0Uih4SllfgtK+svNTeE0r5atMWE0xR489BvsqNuPSxYJUmW28JqhaSZ4SabYrRx114KcU6ko7hkjyPkjQa3P+chStJjIKYgu5tWBiMJp5QVLelKoM+xkY6S7efvJ8AfajxCViLGyDQPDviGr2D0VvIBob0D1ZmAoTvYOWafcNCaqaejPDybFtuLFX3pZBqfyOCyyzGhucyCmfBXJALKbhjRAqN5glNsUmGhhPK87TuGATQfVuZtenMvXMQIDAQAB"
 }

components/brave_extension/extension/resources.grd

@@ -28,6 +28,27 @@
       <include name="IDR_BRAVE_EXTENSION_BRAVELIZER_JS" file="brave_extension/assets/bravelizer.css" type="BINDATA" />
       <include name="IDR_BRAVE_EXTENSION_PICKER_HTML" file="brave_extension/elementPicker.html" type="BINDATA" />
       <include name="IDR_BRAVE_EXTENSION_PICKER_JS" file="brave_extension/elementPicker.js" type="BINDATA" />
+      <include name="IDR_BRAVE_EXTENSION_WEB_DISCOVERY_CONTENT_SCRIPT_BUNDLE"
+               resource_path="brave_extension/bundles/web-discovery-content-script.bundle.js"
+               file="../../brave/vendor/web-discovery-project/build/core/content-script.bundle.js"
+               use_base_dir="false"
+               type="BINDATA" />
+      <!-- WDP extension expects these file paths relative to the extension root -->
+      <include name="IDR_BRAVE_EXTENSION_WEB_DISCOVERY_ASMJS_BUNDLE"
+               resource_path="brave_extension/modules/hpnv2/worker.asmjs.bundle.js"
+               file="../../brave/vendor/web-discovery-project/build/hpnv2/worker.asmjs.bundle.js"
+               use_base_dir="false"
+               type="BINDATA" />
+      <include name="IDR_BRAVE_EXTENSION_WEB_DISCOVERY_WASM_BUNDLE"
+               resource_path="brave_extension/modules/hpnv2/worker.wasm.bundle.js"
+               file="../../brave/vendor/web-discovery-project/build/hpnv2/worker.wasm.bundle.js"
+               use_base_dir="false"
+               type="BINDATA" />
+      <include name="IDR_BRAVE_EXTENSION_WEB_DISCOVERY_STAR_WASM"
+               resource_path="brave_extension/star.wasm"
+               file="../../brave/vendor/web-discovery-project/build/star.wasm"
+               use_base_dir="false"
+               type="BINDATA" />
     </includes>
   </release>
 </grit>

script/brave_license_helper.py

@@ -84,6 +84,11 @@ def AddBraveCredits(prune_paths, special_cases, prune_dirs, additional_paths):
             "URL": "https://github.com/brave/hashset-cpp",
             "License": "MPL-2.0",
         },
+        os.path.join('brave', 'vendor', 'web-discovery-project'): {
+            "Name": "Web Discovery Project",
+            "URL": "https://github.com/brave/web-discovery-project",
+            "License": "MPL-2.0",
+        },
         os.path.join('brave', 'vendor', 'omaha'): {
             "Name": "Omaha",
             "URL": "https://github.com/brave/omaha",
@@ -132,7 +137,7 @@ def AddBraveCredits(prune_paths, special_cases, prune_dirs, additional_paths):
 
 def CheckBraveMissingLicense(target_os, path, error):
     if path.startswith('brave'):
-        if (target_os == 'android'):
+        if target_os == 'android':
             if path in DESKTOP_ONLY_PATHS:
                 return  # Desktop failures are not relevant on Android.
         else: