Use UK pilot Event message in BigQueryEventQuery.

MODULE.bazel

@@ -306,6 +306,13 @@ http_archive(
     url = "https://github.com/google/tink/archive/refs/tags/v1.7.0.tar.gz",
 )
 
+http_archive(
+    name = "uk_pilot_event_templates",
+    sha256 = "e5d9cb0f6087b6286f5bfeb053a968870c88c27fa7a5dc40055ae7681dad9b2c",
+    strip_prefix = "uk-pilot-event-templates-0.2.0",
+    url = "https://github.com/world-federation-of-advertisers/uk-pilot-event-templates/archive/refs/tags/v0.2.0.tar.gz",
+)
+
 # Force use of newer version of boringssl.
 single_version_override(
     module_name = "boringssl",

docs/gke/correctness-test.md

@@ -85,33 +85,14 @@ There are two data sources that can be used for
 
 ### Populate BigQuery table
 
-We can upload pre-generated synthetic test data from the
-[synthetic-labelled-events.csv](../../src/main/k8s/testing/data/synthetic-labelled-events.csv)
-file.
-
-1.  Visit the GCloud console
-    [Bigquery](https://console.cloud.google.com/bigquery) page. Enable the
-    BiqQuery API in the project if you haven't done it yet.
-2.  Create a dataset
-    -   Click the three-dot button to the right of the project
-    -   Click `Create dataset`
-    -   Set the "Dataset ID" to `demo`
-    -   Set the "data location" to (e.g. `us-central1`)
-    -   Click `CREATE DATASET`
-3.  Create a table
-    -   Click the three-dot button to the right of the `demo` data set.
-    -   Click `create table`
-    -   Set "create table from" to `upload`
-    -   Select the local file `halo-cmm-test-labelled-events.csv` in your local
-        branch
-    -   Set Destination Table to `labelled_events`
-    -   Check the Schema -> Auto detect box
-    -   Click `CREATE TABLE`
-    -   You should see a table named `labelled_events` being created. But the
-        `create table` window might still be open, just click `CANCEL` to quit
-        it.
-4.  Check the table is valid
-    -   Click on the `labelled_events` table. You should see something like this
+The BigQuery table can be populated with synthetic event data generated using
+the
+[`uk-pilot-synthetic-data-gen` script](https://github.com/world-federation-of-advertisers/uk-pilot-synthetic-data-gen).
+
+The `dev` configuration expects a table named `labelled_events` in a dataset
+named `demo` in the `us-central1` region. The table can be created in the
+[Google Cloud Console](https://console.cloud.google.com/bigquery), specifying
+the generated CSV file with automatic schema detection.
 
 ![image-step-4-1](step-4-1.png)![image-step-4-1](step-4-2.png)
 

src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt

@@ -123,7 +123,7 @@ class InProcessEdpSimulator(
     edpJob.cancelAndJoin()
   }
 
-  suspend fun ensureEventGroup() = delegate.ensureEventGroup(syntheticDataSpec)
+  suspend fun ensureEventGroup() = delegate.ensureEventGroup(EVENT_TEMPLATES, syntheticDataSpec)
 
   /** Builds a [EdpData] object for the Edp with a certain [displayName] and [resourceName]. */
   @Blocking
@@ -139,7 +139,10 @@ class InProcessEdpSimulator(
 
   companion object {
     private val logger: Logger = Logger.getLogger(this::class.java.name)
+
     private const val RANDOM_SEED: Long = 1
     private val random = Random(RANDOM_SEED)
+
+    private val EVENT_TEMPLATES = EdpSimulator.buildEventTemplates(TestEvent.getDescriptor())
   }
 }

src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BUILD.bazel

@@ -65,7 +65,9 @@ kt_jvm_library(
         ":event_query",
         ":labeled_event",
         "//src/main/kotlin/org/wfanet/measurement/eventdataprovider/eventfiltration:event_filters",
-        "//src/main/proto/wfa/measurement/api/v2alpha/event_templates/testing:test_event_kt_jvm_proto",
+        "//src/main/proto/halo_cmm/uk/pilot:display_template_kt_jvm_proto",
+        "//src/main/proto/halo_cmm/uk/pilot:event_kt_jvm_proto",
+        "//src/main/proto/halo_cmm/uk/pilot:video_template_kt_jvm_proto",
         "@wfa_common_jvm//imports/java/com/google/cloud/bigquery",
         "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common",
         "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/common",
@@ -149,6 +151,7 @@ kt_jvm_library(
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:privacy_budgets",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:test_identifiers",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:vid_sampling",
+        "//src/main/proto/halo_cmm/uk/pilot:event_kt_jvm_proto",
         "//src/main/proto/wfa/any_sketch:sketch_kt_jvm_proto",
         "//src/main/proto/wfa/measurement/api/v2alpha:certificates_service_kt_jvm_grpc_proto",
         "//src/main/proto/wfa/measurement/api/v2alpha:crypto_kt_jvm_proto",
@@ -227,6 +230,7 @@ kt_jvm_library(
         ":csv_event_query",
         ":edp_simulator_runner",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:event_group_metadata",
+        "//src/main/proto/wfa/measurement/api/v2alpha/event_templates/testing:test_event_kt_jvm_proto",
         "@wfa_common_jvm//imports/java/picocli",
         "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common",
     ],
@@ -258,6 +262,7 @@ kt_jvm_library(
         ":bigquery_event_query",
         ":edp_simulator_runner",
         "//src/main/kotlin/org/wfanet/measurement/loadtest/config:event_group_metadata",
+        "//src/main/proto/halo_cmm/uk/pilot:event_kt_jvm_proto",
         "@wfa_common_jvm//imports/java/picocli",
         "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common",
     ],

src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt

@@ -17,6 +17,7 @@ package org.wfanet.measurement.loadtest.dataprovider
 import com.google.cloud.bigquery.BigQuery
 import com.google.cloud.bigquery.BigQueryOptions
 import kotlin.properties.Delegates
+import org.halo_cmm.uk.pilot.Event
 import org.wfanet.measurement.common.commandLineMain
 import org.wfanet.measurement.loadtest.config.EventGroupMetadata
 import picocli.CommandLine
@@ -71,7 +72,11 @@ class BigQueryEdpSimulatorRunner : EdpSimulatorRunner() {
         publisherId,
       )
 
-    run(eventQuery, mapOf("" to EventGroupMetadata.testMetadata(publisherId)))
+    run(
+      eventQuery,
+      EdpSimulator.buildEventTemplates(Event.getDescriptor()),
+      mapOf("" to EventGroupMetadata.testMetadata(publisherId)),
+    )
   }
 }
 

src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEventQuery.kt

@@ -27,12 +27,11 @@ import java.time.Instant
 import java.time.temporal.ChronoUnit
 import java.util.UUID
 import java.util.logging.Logger
+import org.halo_cmm.uk.pilot.Event
+import org.halo_cmm.uk.pilot.display
+import org.halo_cmm.uk.pilot.event
+import org.halo_cmm.uk.pilot.video
 import org.wfanet.measurement.api.v2alpha.EventGroup
-import org.wfanet.measurement.api.v2alpha.event_templates.testing.Person
-import org.wfanet.measurement.api.v2alpha.event_templates.testing.TestEvent
-import org.wfanet.measurement.api.v2alpha.event_templates.testing.person
-import org.wfanet.measurement.api.v2alpha.event_templates.testing.testEvent
-import org.wfanet.measurement.api.v2alpha.event_templates.testing.video
 import org.wfanet.measurement.common.OpenEndTimeRange
 import org.wfanet.measurement.common.toRange
 import org.wfanet.measurement.eventdataprovider.eventfiltration.EventFilters
@@ -43,12 +42,12 @@ abstract class BigQueryEventQuery(
   private val bigQuery: BigQuery,
   private val datasetName: String,
   private val tableName: String,
-) : EventQuery<TestEvent> {
+) : EventQuery<Event> {
   protected abstract fun getPublisherId(eventGroup: EventGroup): Int
 
   override fun getLabeledEvents(
     eventGroupSpec: EventQuery.EventGroupSpec
-  ): Sequence<LabeledEvent<TestEvent>> {
+  ): Sequence<LabeledEvent<Event>> {
     val timeRange: OpenEndTimeRange = eventGroupSpec.spec.collectionInterval.toRange()
     val queryConfig: QueryJobConfiguration =
       buildQueryConfig(getPublisherId(eventGroupSpec.eventGroup), timeRange)
@@ -66,7 +65,7 @@ abstract class BigQueryEventQuery(
     }
     logger.info("Running query on BigQuery table.")
 
-    val program = EventQuery.compileProgram(eventGroupSpec.spec.filter, TestEvent.getDescriptor())
+    val program = EventQuery.compileProgram(eventGroupSpec.spec.filter, Event.getDescriptor())
 
     return resultJob
       .getQueryResults()
@@ -102,52 +101,68 @@ abstract class BigQueryEventQuery(
       .build()
   }
 
-  private fun FieldValueList.toLabeledEvent(): LabeledEvent<TestEvent> {
-    val gender: Person.Gender? =
-      when (get("sex").stringValue) {
-        "M" -> Person.Gender.MALE
-        "F" -> Person.Gender.FEMALE
-        else -> null
-      }
-    val ageGroup: Person.AgeGroup? =
-      when (get("age_group").stringValue) {
-        "18_34" -> Person.AgeGroup.YEARS_18_TO_34
-        "35_54" -> Person.AgeGroup.YEARS_35_TO_54
-        "55+" -> Person.AgeGroup.YEARS_55_PLUS
-        else -> null
-      }
-    val socialGradeGroup: Person.SocialGradeGroup? =
-      when (get("social_grade").stringValue) {
-        "ABC1" -> Person.SocialGradeGroup.A_B_C1
-        "C2DE" -> Person.SocialGradeGroup.C2_D_E
-        else -> null
-      }
-    val complete: Boolean =
-      when (get("complete").longValue) {
-        0L -> false
-        else -> true
-      }
-    val message = testEvent {
-      person = person {
-        if (gender != null) {
-          this.gender = gender
-        }
-        if (ageGroup != null) {
-          this.ageGroup = ageGroup
+  private fun FieldValueList.toLabeledEvent(): LabeledEvent<Event> {
+    val viewability = get("viewability").stringValue
+    val completionStatus = get("digital_video_completion_status").stringValue
+    val message = event {
+      video = video {
+        when (completionStatus) {
+          "0% - 25%" -> completed0PercentPlus = true
+          "25% - 50%" -> {
+            completed0PercentPlus = true
+            completed25PercentPlus = true
+          }
+          "50% - 75%" -> {
+            completed0PercentPlus = true
+            completed25PercentPlus = true
+            completed50PercentPlus = true
+          }
+          "75% - 100%" -> {
+            completed0PercentPlus = true
+            completed25PercentPlus = true
+            completed50PercentPlus = true
+            completed75PercentPlus = true
+          }
+          "100%" -> {
+            completed0PercentPlus = true
+            completed25PercentPlus = true
+            completed50PercentPlus = true
+            completed75PercentPlus = true
+            completed100Percent = true
+          }
+          else -> error("Unknown video completion status :  $completionStatus")
         }
-        if (socialGradeGroup != null) {
-          this.socialGradeGroup = socialGradeGroup
+        when (viewability) {
+          "viewable_0_percent_to_50_percent" -> viewable0PercentPlus = true
+          "viewable_50_percent_to_100_percent" -> {
+            viewable0PercentPlus = true
+            viewable50PercentPlus = true
+          }
+          "viewable_100_percent" -> {
+            viewable0PercentPlus = true
+            viewable50PercentPlus = true
+            viewable100Percent = true
+          }
+          else -> error("Unknown video viewability :  $viewability")
         }
       }
-      videoAd = video {
-        viewedFraction =
-          if (complete) {
-            1.0
-          } else {
-            0.0
+      display = display {
+        when (viewability) {
+          "viewable_0_percent_to_50_percent" -> viewable0PercentPlus = true
+          "viewable_50_percent_to_100_percent" -> {
+            viewable0PercentPlus = true
+            viewable50PercentPlus = true
           }
+          "viewable_100_percent" -> {
+            viewable0PercentPlus = true
+            viewable50PercentPlus = true
+            viewable100Percent = true
+          }
+          else -> throw Exception("Unknown display viewability :  $viewability")
+        }
       }
     }
+
     return LabeledEvent(
       Timestamp.ofTimeMicroseconds(get("time").timestampValue).toInstant(),
       get("vid").longValue,

src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt

@@ -16,6 +16,7 @@ package org.wfanet.measurement.loadtest.dataprovider
 
 import java.io.File
 import kotlin.properties.Delegates
+import org.wfanet.measurement.api.v2alpha.event_templates.testing.TestEvent
 import org.wfanet.measurement.common.commandLineMain
 import org.wfanet.measurement.loadtest.config.EventGroupMetadata
 import picocli.CommandLine
@@ -47,7 +48,11 @@ class CsvEdpSimulatorRunner : EdpSimulatorRunner() {
 
   override fun run() {
     val eventQuery = CsvEventQuery(publisherId, eventsCsv)
-    run(eventQuery, mapOf("" to EventGroupMetadata.testMetadata(publisherId)))
+    run(
+      eventQuery,
+      EdpSimulator.buildEventTemplates(TestEvent.getDescriptor()),
+      mapOf("" to EventGroupMetadata.testMetadata(publisherId)),
+    )
   }
 }