Webcompat sightline

.pre-commit-config.yaml

@@ -61,5 +61,3 @@ repos:
   - repo: meta
     hooks:
       - id: check-useless-excludes
-default_language_version:
-  python: python3.10

bugbot/gcp.py

@@ -0,0 +1,36 @@
+from typing import Iterable, Optional
+
+from google.cloud import bigquery
+from google.oauth2 import service_account
+
+from bugbot import utils
+
+SCOPES = {
+    "cloud-platform": "https://www.googleapis.com/auth/cloud-platform",
+    "drive": "https://www.googleapis.com/auth/drive",
+}
+
+
+def get_bq_client(
+    project: str, scopes: Optional[Iterable[str]] = None
+) -> bigquery.Client:
+    """Get a bigquery.Client for a given project
+
+    :param project: Name of the project.
+    :param scopes: Optional iterable containing the scopes the client should have.
+                   By default this will be the cloud-platform scopes required to
+                   run queries.
+    :returns: bigquery.Client
+    """
+    scope_urls = []
+    if scopes is None:
+        scope_urls.append(SCOPES["cloud-platform"])
+    else:
+        for item in scopes:
+            scope_urls.append(SCOPES[item] if item in SCOPES else item)
+
+    credentials = service_account.Credentials.from_service_account_info(
+        utils.get_gcp_service_account_info()
+    ).with_scopes(scope_urls)
+
+    return bigquery.Client(project=project, credentials=credentials)

bugbot/rules/webcompat_platform_without_keyword.py

@@ -2,69 +2,44 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from libmozdata.bugzilla import Bugzilla
-
+from bugbot import gcp
 from bugbot.bzcleaner import BzCleaner
 
 
 class WebcompatPlatformWithoutKeyword(BzCleaner):
     normal_changes_max = 200
 
     def description(self):
-        return "Core bugs blocking webcompat knowledge base entries without webcompat:platform-bug"
+        return "Web Compat platform bugs without webcompat:platform-bug keyword"
 
     def filter_no_nag_keyword(self):
         return False
 
     def has_default_products(self):
         return False
 
-    def handle_bug(self, bug, data):
-        data[bug["id"]] = {"depends_on": set(bug.get("depends_on", []))}
-        return bug
-
-    def get_core_bugs(self, bugs):
-        core_bugs = set()
-        for bug_data in bugs.values():
-            core_bugs |= bug_data.get("depends_on", set())
-
-        def bug_handler(bug, data):
-            if "webcompat:platform-bug" not in bug["keywords"]:
-                data[bug["id"]] = bug
-
-        core_bug_data = {}
-
-        Bugzilla(
-            bugids=list(core_bugs),
-            include_fields=["id", "summary", "keywords"],
-            bughandler=bug_handler,
-            bugdata=core_bug_data,
-        ).get_data().wait()
-
-        return core_bug_data
-
     def get_autofix_change(self):
         return {
             "keywords": {"add": ["webcompat:platform-bug"]},
         }
 
     def get_bz_params(self, date):
-        fields = [
-            "id",
-            "depends_on",
-        ]
-        params = {
-            "include_fields": fields,
-            "product": "Web Compatibility",
-            "component": "Knowledge Base",
-        }
-
-        return params
-
-    def get_bugs(self, *args, **kwargs):
-        bugs = super().get_bugs(*args, **kwargs)
-        bugs = self.get_core_bugs(bugs)
-        return bugs
+        fields = ["id", "summary", "keywords"]
+        return {"include_fields": fields, "id": self.get_core_bug_ids()}
+
+    def get_core_bug_ids(self):
+        project = "moz-fx-dev-dschubert-wckb"
+        dataset = "webcompat_knowledge_base"
+
+        client = gcp.get_client(project, ["cloud-platform", "drive"])
+        query = f"""
+        SELECT core_bug FROM `{project}.{dataset}.prioritized_kb_entries` as kb_entries
+        JOIN `moz-fx-dev-dschubert-wckb.webcompat_knowledge_base.bugzilla_bugs` as bugzilla_bugs ON bugzilla_bugs.number = kb_entries.core_bug
+        WHERE "webcompat:platform-bug" not in UNNEST(bugzilla_bugs.keywords)
+        LIMIT {self.normal_changes_max}
+        """
+
+        return list(row["core_bug"] for row in client.query(query).result())
 
 
 if __name__ == "__main__":

bugbot/rules/webcompat_sightline.py

@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from typing import Any, Optional
+
+from bugbot import gcp
+from bugbot.bzcleaner import BzCleaner
+
+
+class WebcompatSightline(BzCleaner):
+    normal_changes_max = 1000
+    whiteboard_entry = "[webcompat:sightline]"
+
+    def __init__(self):
+        super().__init__()
+        self.autofix_changes = {}
+        self.sightline_ids = []
+
+    def description(self) -> str:
+        return "Web Compat site report in the sightline metric set"
+
+    def filter_no_nag_keyword(self) -> bool:
+        return False
+
+    def has_default_products(self) -> bool:
+        return False
+
+    def get_autofix_change(self) -> dict[str, Any]:
+        return self.autofix_changes
+
+    def handle_bug(
+        self, bug: dict[str, Any], data: dict[str, Any]
+    ) -> Optional[dict[str, Any]]:
+        bug_id = str(bug["id"])
+        whiteboard = bug["whiteboard"]
+
+        if bug["id"] in self.sightline_ids:
+            if self.whiteboard_entry not in whiteboard:
+                self.autofix_changes[bug_id] = {
+                    "whiteboard": whiteboard + self.whiteboard_entry
+                }
+        elif self.whiteboard_entry in whiteboard:
+            self.autofix_changes[bug_id] = {
+                "whiteboard": whiteboard.replace(self.whiteboard_entry, "")
+            }
+
+        return None
+
+    def get_bz_params(self, date) -> dict[str, Any]:
+        fields = ["id", "summary", "whiteboard"]
+        self.sightline_ids = self.get_bug_ids()
+        # Get all bugs that either have, or should have, the [webcompat:sightline]
+        # whiteboard entry
+        return {
+            "include_fields": fields,
+            "j_top": "OR",
+            "f1": "bug_id",
+            "o1": "anyexact",
+            "v1": ",".join(str(item) for item in self.sightline_ids),
+            "f2": "status_whiteboard",
+            "o2": "substring",
+            "v2": self.whiteboard_entry,
+        }
+
+    def get_bug_ids(self) -> list[int]:
+        project = "moz-fx-dev-dschubert-wckb"
+        dataset = "webcompat_knowledge_base"
+
+        client = gcp.get_bq_client(project, ["cloud-platform", "drive"])
+        query = f"""
+        SELECT number FROM `{project}.{dataset}.webcompat_topline_metric_site_reports` as bugs
+        """
+
+        return list(row["number"] for row in client.query(query).result())
+
+
+if __name__ == "__main__":
+    WebcompatSightline().run()

requirements.txt

@@ -1,6 +1,7 @@
 alembic==1.13.3
 argparse==1.4.0
 filelock==3.16.1
+google-cloud-bigquery==3.24.0
 gspread==6.1.2
 humanize>=0.5.1
 icalendar==5.0.13

scripts/cron_run_hourly.sh

@@ -75,4 +75,7 @@ python -m bugbot.rules.duplicate_copy_metadata --production
 # Add `webcompat:platform-bug` keyword to bugs without a platform keyword
 python -m bugbot.rules.webcompat_platform_without_keyword --production
 
+# Add `[webcompat:sightline]` whiteboard entry to bugs in sightline metric set
+python -m bugbot.rules.webcompat_sightline --production
+
 source ./scripts/cron_common_end.sh