Privacy 2024 queries

sql/2024/privacy/ads_and_sellers_graph.sql

@@ -0,0 +1,106 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    CASE page -- publisher websites may redirect to an SSP domain, and need to use redirected domain instead of page domain
+      WHEN 'https://www.chunkbase.com/' THEN 'cafemedia.com'
+      ELSE NET.REG_DOMAIN(page)
+    END AS page,
+    custom_metrics
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    client = 'desktop' AND
+    is_root_page = TRUE AND
+    rank <= 10000
+), ads AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.ads.account_types') AS ad_accounts
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_count') AS INT64) > 0
+), sellers AS (
+  SELECT
+    page,
+    JSON_QUERY(custom_metrics, '$.ads.sellers.seller_types') AS ad_sellers
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(custom_metrics, '$.ads.sellers.seller_count') AS INT64) > 0
+), relationships AS (
+  SELECT
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    page AS publisher
+  FROM ads, UNNEST(JSON_VALUE_ARRAY(ad_accounts, '$.direct.domains')) AS domain
+  UNION ALL
+  SELECT
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS demand,
+    'Web' AS supply,
+    'indirect' AS relationship,
+    page AS publisher
+  FROM ads, UNNEST(JSON_VALUE_ARRAY(ad_accounts, '$.reseller.domains')) AS domain
+  UNION ALL
+  SELECT
+    page AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.publisher.domains')) AS domain
+  UNION ALL
+  SELECT
+    page AS demand,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS supply,
+    'indirect' AS relationship,
+    NULL AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.intermediary.domains')) AS domain
+  UNION ALL
+  SELECT
+    page AS demand,
+    'Web' AS supply,
+    'direct' AS relationship,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.both.domains')) AS domain
+  UNION ALL
+  SELECT
+    page AS demand,
+    NET.REG_DOMAIN(REGEXP_EXTRACT(NORMALIZE_AND_CASEFOLD(domain), r'\b[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')) AS supply,
+    'indirect' AS relationship,
+    NULL AS publisher
+  FROM sellers, UNNEST(JSON_VALUE_ARRAY(ad_sellers, '$.both.domains')) AS domain
+), nodes AS (
+  (
+    SELECT
+      demand,
+      supply,
+      CONCAT(demand, '-', supply) AS path_history,
+      relationship,
+      HLL_COUNT.INIT(publisher) AS supply_sketch
+    FROM relationships
+    WHERE supply = 'Web'
+    GROUP BY demand, supply, relationship
+  )
+  UNION ALL
+  (
+    SELECT
+      relationships.demand AS demand,
+      relationships.supply AS supply,
+      CONCAT(relationships.demand, '-', nodes.path_history) AS path_history,
+      relationships.relationship AS relationship,
+      nodes.supply_sketch AS supply_sketch
+    FROM relationships
+    INNER JOIN nodes
+    ON relationships.supply = nodes.demand AND
+      nodes.supply_sketch IS NOT NULL AND
+      nodes.relationship = 'indirect' AND
+      STRPOS(nodes.path_history, CONCAT(relationships.demand, '-', relationships.supply)) = 0
+  )
+)
+
+SELECT
+  demand,
+  supply,
+  path_history,
+  relationship,
+  HLL_COUNT.MERGE(supply_sketch) AS publishers_count
+FROM nodes
+GROUP BY demand, supply, relationship, path_history
+ORDER BY publishers_count DESC

sql/2024/privacy/ads_lines_amount.sql

@@ -0,0 +1,27 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    CASE page -- publisher websites may redirect to an SSP domain, and need to use redirected domain instead of page domain
+      WHEN 'https://www.chunkbase.com/' THEN 'cafemedia.com'
+      ELSE NET.REG_DOMAIN(page)
+    END AS page,
+    custom_metrics
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    client = 'desktop' AND
+    is_root_page = TRUE AND
+    rank <= 10000
+), ads AS (
+  SELECT
+    page,
+    CEIL(CAST(JSON_VALUE(custom_metrics, '$.ads.ads.line_count') AS INT64) / 100) * 100 AS line_count_bucket
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.line_count') AS INT64) > 0
+)
+
+SELECT
+  line_count_bucket,
+  COUNT(DISTINCT page) AS page_count
+FROM ads
+GROUP BY line_count_bucket
+ORDER BY line_count_bucket ASC

sql/2024/privacy/ads_seller_accounts_by_type.sql

@@ -0,0 +1,33 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    NET.REG_DOMAIN(page) AS page,
+    custom_metrics
+  FROM `httparchive.all.pages`
+  WHERE date = '2024-06-01' AND
+    client = 'desktop' AND
+    is_root_page = TRUE AND
+    rank <= 10000
+), ads AS (
+  SELECT
+    CEIL(CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_types.direct.account_count') AS INT64) / 100) * 100 AS direct_account_count_bucket,
+    CEIL(CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_types.reseller.account_count') AS INT64) / 100) * 100 AS reseller_account_count_bucket
+  FROM pages
+  WHERE
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_count') AS INT64) > 0
+)
+
+SELECT
+  'direct' AS account_type,
+  direct_account_count_bucket AS account_count_bucket,
+  COUNT(0) AS page_count
+FROM ads
+GROUP BY direct_account_count_bucket
+UNION ALL
+SELECT
+  'reseller' AS account_type,
+  reseller_account_count_bucket AS account_count_bucket,
+  COUNT(0) AS page_count
+FROM ads
+GROUP BY reseller_account_count_bucket
+ORDER BY account_count_bucket ASC
+LIMIT 200

sql/2024/privacy/ara-trigger-registrations-for-different-destinations-by-destinations.sql

@@ -0,0 +1,78 @@
+-- ara-trigger-registrations-for-different-destinations-by-destinations.sql
+-- Analysis of Attribution Reporting API (ARA) Triggers registered for different destinations by destination domains (i.e., advertiser domains):
+-- 1. No. of third-parties that register trigger for the given destination
+-- 2. Min. epsilon -- MIN(CASE WHEN epsilon IS NOT NULL THEN epsilon END) AS min_epsilon
+-- 3. Avg. epsilon -- AVG(CASE WHEN epsilon IS NOT NULL THEN epsilon END) AS avg_epsilon
+-- 4. Max. epsilon -- MAX(CASE WHEN epsilon IS NOT NULL THEN epsilon END) AS max_epsilon
+-- [Higher the epsilon, the more the privacy protection] [Epsilon is always undefined, so last 3 columns are removed for this year atleast]
+-- Output comprises 9.5K rows and 1 column.
+
+-- Extracting third-parties observed using ARA API on a publisher
+CREATE TEMP FUNCTION jsonObjectKeys(input STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
+  if (!input) {
+    return [];
+  }
+  return Object.keys(JSON.parse(input));
+""";
+
+-- Extracting ARA API source registration details being passed by a given third-party (passed as "key")
+CREATE TEMP FUNCTION jsonObjectValues(input STRING, key STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
+  if (!input) {
+    return [];
+  }
+  const jsonObject = JSON.parse(input);
+  const values = jsonObject[key] || [];
+  const result = [];
+
+  values.forEach(value => {
+    if (value.toLowerCase().startsWith('attribution-reporting-register-source|')) {
+      const parts = value.replace('attribution-reporting-register-source|', '').split('|');
+      parts.forEach(part => {
+        if (part.startsWith('destination=')) {
+          const destinations = part.replace('destination=', '').split(',');
+          destinations.forEach(destination => {
+            result.push('destination=' + destination.trim());
+          });
+        } else {
+          result.push(part.trim());
+        }
+      });
+    }
+  });
+
+  return result;
+""";
+
+WITH ara_features AS (
+  SELECT
+    NET.REG_DOMAIN(page) AS publisher,
+    third_party_domain,
+    CASE
+      WHEN ara LIKE 'destination=%' THEN NET.REG_DOMAIN(REPLACE(ara, 'destination=', ''))
+      ELSE NULL
+    END AS destination,
+    CASE
+      WHEN ara LIKE 'epsilon=%' THEN SAFE_CAST(REPLACE(ara, 'epsilon=', '') AS FLOAT64)
+      ELSE NULL
+    END AS epsilon
+  FROM `httparchive.all.pages`,
+    UNNEST(jsonObjectKeys(JSON_QUERY(custom_metrics, '$.privacy-sandbox.privacySandBoxAPIUsage'))) AS third_party_domain,
+    UNNEST(jsonObjectValues(JSON_QUERY(custom_metrics, '$.privacy-sandbox.privacySandBoxAPIUsage'), third_party_domain)) AS ara
+  WHERE
+    date = '2024-06-01' AND
+    client = 'desktop' AND
+    is_root_page = TRUE AND
+    rank <= 1000000
+)
+SELECT
+  destination,
+  COUNT(DISTINCT third_party_domain) AS third_party_count
+FROM ara_features
+WHERE destination IS NOT NULL
+GROUP BY destination
+HAVING third_party_count > 0
+ORDER BY third_party_count DESC;

sql/2024/privacy/ara-trigger-registrations-for-different-destinations-by-third-parties.sql

@@ -0,0 +1,78 @@
+-- ara-trigger-registrations-for-different-destinations-by-third-parties.sql
+-- Analysis of Attribution Reporting API (ARA) Triggers registered for different destinations by Third Party (TP) domains:
+-- 1. No. of destinations registered by a given TP
+-- 2. Min. epsilon -- MIN(CASE WHEN epsilon IS NOT NULL THEN epsilon END) AS min_epsilon
+-- 3. Avg. epsilon -- AVG(CASE WHEN epsilon IS NOT NULL THEN epsilon END) AS avg_epsilon
+-- 4. Max. epsilon -- MAX(CASE WHEN epsilon IS NOT NULL THEN epsilon END) AS max_epsilon
+-- [Higher the epsilon, the more the privacy protection] [Epsilon is always undefined, so last 3 columns are removed for this year]
+-- Output comprises 17 rows and 1 column.
+
+-- Extracting third-parties observed using ARA API on a publisher
+CREATE TEMP FUNCTION jsonObjectKeys(input STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
+  if (!input) {
+    return [];
+  }
+  return Object.keys(JSON.parse(input));
+""";
+
+-- Extracting ARA API source registration details being passed by a given third-party (passed as "key")
+CREATE TEMP FUNCTION jsonObjectValues(input STRING, key STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
+  if (!input) {
+    return [];
+  }
+  const jsonObject = JSON.parse(input);
+  const values = jsonObject[key] || [];
+  const result = [];
+
+  values.forEach(value => {
+    if (value.toLowerCase().startsWith('attribution-reporting-register-source|')) {
+      const parts = value.replace('attribution-reporting-register-source|', '').split('|');
+      parts.forEach(part => {
+        if (part.startsWith('destination=')) {
+          const destinations = part.replace('destination=', '').split(',');
+          destinations.forEach(destination => {
+            result.push('destination=' + destination.trim());
+          });
+        } else {
+          result.push(part.trim());
+        }
+      });
+    }
+  });
+
+  return result;
+""";
+
+WITH ara_features AS (
+  SELECT
+    NET.REG_DOMAIN(page) AS publisher,
+    third_party_domain,
+    CASE
+      WHEN ara LIKE 'destination=%' THEN NET.REG_DOMAIN(REPLACE(ara, 'destination=', ''))
+      ELSE NULL
+    END AS destination,
+    CASE
+      WHEN ara LIKE 'epsilon=%' THEN SAFE_CAST(REPLACE(ara, 'epsilon=', '') AS FLOAT64)
+      ELSE NULL
+    END AS epsilon
+  FROM `httparchive.all.pages`,
+    UNNEST(jsonObjectKeys(JSON_QUERY(custom_metrics, '$.privacy-sandbox.privacySandBoxAPIUsage'))) AS third_party_domain,
+    UNNEST(jsonObjectValues(JSON_QUERY(custom_metrics, '$.privacy-sandbox.privacySandBoxAPIUsage'), third_party_domain)) AS ara
+  WHERE
+    date = '2024-06-01' AND
+    client = 'desktop' AND
+    is_root_page = TRUE AND
+    rank <= 1000000
+)
+SELECT
+  third_party_domain,
+  COUNT(DISTINCT destination) AS destination_count
+FROM ara_features
+WHERE third_party_domain IS NOT NULL
+GROUP BY third_party_domain
+HAVING destination_count > 0
+ORDER BY destination_count DESC;

sql/2024/privacy/attested-domains-in-top-1M-using-privacy-sandbox.sql

@@ -0,0 +1,31 @@
+-- attested-domains-in-top-1M-using-privacy-sandbox.sql
+-- Contributed by: @yohhaan
+-- Urls that may have a `/.well-known/related-website-set.json` for Privacy Sandbox Related Website Set Proposal and
+-- Urls that may have a `/.well-known/privacy-sandbox-attestations.json` for Privacy Sandbox APIs Attestation file
+-- Note: we are only extracting a potential list of origins that may have the file to feed to another crawler (https://github.com/privacysandstorm/well-known-crawler) that will actually check if file is valid by parsing it
+-- Test query on `httparchive.sample_data.pages_1k` and `TABLESAMPLE SYSTEM (1.0 PERCENT)` with latest date of crawl
+-- Final query on `httparchive.all.pages`
+
+
+WITH wellknown AS (
+  SELECT
+    NET.HOST(page) AS host,
+    CAST(JSON_VALUE(custom_metrics, '$.well-known."/.well-known/related-website-set.json".found') AS BOOL) AS rws,
+    CAST(JSON_VALUE(custom_metrics, '$.well-known."/.well-known/privacy-sandbox-attestations.json".found') AS BOOL) AS attestation
+  FROM
+    `httparchive.all.pages`
+  WHERE
+    date = '2024-06-01' AND
+    client = 'desktop' AND
+    is_root_page = TRUE AND
+    rank <= 1000000
+)
+
+SELECT DISTINCT
+  host,
+  CASE WHEN rws THEN 1 ELSE 0 END AS related_websites_set,
+  CASE WHEN attestation THEN 1 ELSE 0 END AS privacy_sandbox_attestation
+FROM
+  wellknown
+WHERE
+  rws OR attestation;

sql/2024/privacy/common_ads_variables.sql

@@ -0,0 +1,29 @@
+WITH RECURSIVE pages AS (
+  SELECT
+    NET.REG_DOMAIN(page) AS page,
+    custom_metrics
+  FROM `httparchive.all.pages`
+  WHERE
+    date = '2024-06-01' AND
+    client = 'desktop' AND
+    is_root_page = TRUE AND
+    rank <= 10000
+), ads AS (
+  SELECT
+    page,
+    variable,
+    COUNT(DISTINCT page) OVER() AS total_publishers
+  FROM pages,
+    UNNEST(JSON_VALUE_ARRAY(custom_metrics, '$.ads.ads.variables')) AS variable
+  WHERE
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_types.reseller.account_count') AS INT64) > 0 OR
+    CAST(JSON_VALUE(custom_metrics, '$.ads.ads.account_types.direct.account_count') AS INT64) > 0
+)
+
+SELECT
+  variable,
+  COUNT(DISTINCT page) AS publishers_count,
+  ANY_VALUE(total_publishers) AS total_publishers
+FROM ads
+GROUP BY variable
+ORDER BY publishers_count DESC