Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for protobuf Duration and Timestamp fields in synthetic generator. #1145

Merged
merged 1 commit into from
Aug 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ object SyntheticDataGeneration {
FieldValue.ValueCase.FLOAT_VALUE -> fieldValue.floatValue
FieldValue.ValueCase.INT32_VALUE -> fieldValue.int32Value
FieldValue.ValueCase.INT64_VALUE -> fieldValue.int64Value
FieldValue.ValueCase.DURATION_VALUE -> fieldValue.durationValue
FieldValue.ValueCase.TIMESTAMP_VALUE -> fieldValue.timestampValue
FieldValue.ValueCase.VALUE_NOT_SET -> throw IllegalArgumentException()
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ proto_library(
strip_import_prefix = IMPORT_PREFIX,
deps = [
"@com_google_googleapis//google/type:date_proto",
"@com_google_protobuf//:duration_proto",
"@com_google_protobuf//:timestamp_proto",
],
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,20 @@ syntax = "proto3";

package wfa.measurement.api.v2alpha.event_group_metadata.testing;

import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
import "google/type/date.proto";

option java_package = "org.wfanet.measurement.api.v2alpha.event_group_metadata.testing";
option java_multiple_files = true;

// The specification of synthetic EventGroups created from a shared synthetic
// virtual population.
message SimulatorSyntheticDataSpec {
repeated SyntheticEventGroupSpec event_group_spec = 1;
SyntheticPopulationSpec population = 2;
}

// A sequence of VIDs represented with a beginning and exclusive end.
message VidRange {
int64 start = 1;
int64 end_exclusive = 2;
}

// Value of an event field.
message FieldValue {
oneof value {
string string_value = 1;
Expand All @@ -44,24 +40,38 @@ message FieldValue {
float float_value = 5;
int32 int32_value = 6;
int64 int64_value = 7;
google.protobuf.Duration duration_value = 8;
google.protobuf.Timestamp timestamp_value = 9;
}
}

// The specification of a synthetic virtual population.
// The specification of a population for synthetic event generation.
message SyntheticPopulationSpec {
// The overall range of VIDs for the synthetic population.
VidRange vid_range = 1;

// Set of strings that define population data fields such as Age, Gender,
// and Social Grade. These should conform to a CEL expression syntax (e.g.
// person.age_group). These are assigned at the subpopulation level.
// Set of field paths within a synthetic event that pertain to the population,
// with `.` as the traversal operator.
//
// For example, an `age_group` field within a `person` event template would
// have the path `person.age_group`.
//
// The values of these fields are assigned at the sub-population level.
//
// TODO(@SanjayVas): Consider determining this set from protobuf options on
// event template definitions.
repeated string population_fields = 2;

// Set of strings that define non-population data fields such as Device,
// Location, and Duration. These should conform to a CEL expression syntax
// (e.g.person.age_group). These are assigned at the impression level in the
// FrequencySpec.
// Set of field paths within a synthetic event that do not pertain to the
// population, with `.` as the traversal operator.
//
// For example, a `viewed_fraction` field within a `video_ad` event template
// would have the path `video_ad.viewed_fraction`.
//
// The values of these fields are assigned at the event level.
repeated string non_population_fields = 3;

// A subset of the synthetic population.
message SubPopulation {
VidRange vid_sub_range = 1;

Expand All @@ -74,8 +84,10 @@ message SyntheticPopulationSpec {
repeated SubPopulation sub_populations = 4;
}

// The specification of a synthetic EventGroup which describes all impressions
// for specific dates.
// The specification of an `EventGroup` for synthetic event generation.
//
// Each `SyntheticEventGroupSpec` implicitly references a single
// `SyntheticPopulationSpec`.
message SyntheticEventGroupSpec {
string description = 1;

Expand All @@ -84,8 +96,9 @@ message SyntheticEventGroupSpec {
message FrequencySpec {
int64 frequency = 1;

// The specification of non_population_values for a VID range.
// The specification of non-population field values for a VID range.
message VidRangeSpec {
// A range of VIDs within a single `SubPopulation`.
VidRange vid_range = 1;

// A map of `non_population_fields` from `SyntheticPopulationSpec` to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package org.wfanet.measurement.loadtest.dataprovider

import com.google.common.truth.Truth.assertThat
import com.google.type.date
import java.time.Duration
import java.time.LocalDate
import java.time.ZoneOffset
import kotlin.test.assertFailsWith
Expand All @@ -36,6 +37,7 @@ import org.wfanet.measurement.api.v2alpha.event_templates.testing.banner
import org.wfanet.measurement.api.v2alpha.event_templates.testing.person
import org.wfanet.measurement.api.v2alpha.event_templates.testing.testEvent
import org.wfanet.measurement.api.v2alpha.event_templates.testing.video
import org.wfanet.measurement.common.toProtoDuration

@RunWith(JUnit4::class)
class SyntheticDataGenerationTest {
Expand Down Expand Up @@ -261,6 +263,71 @@ class SyntheticDataGenerationTest {
assertThat(parsedLabeledEvents).containsExactlyElementsIn(expectedTestEvents)
}

@Test
fun `generateEvents returns messages with a Duration field`() {
val populationSpec = syntheticPopulationSpec {
vidRange = vidRange {
start = 1L
endExclusive = 11L
}
populationFields += "person.gender"
nonPopulationFields += "video_ad.length"

subPopulations +=
SyntheticPopulationSpecKt.subPopulation {
vidSubRange = [email protected]
populationFieldsValues["person.gender"] = fieldValue {
enumValue = Person.Gender.FEMALE_VALUE
}
}
}
val videoLength = Duration.ofMinutes(5).toProtoDuration()
val eventGroupSpec = syntheticEventGroupSpec {
dateSpecs +=
SyntheticEventGroupSpecKt.dateSpec {
dateRange =
SyntheticEventGroupSpecKt.DateSpecKt.dateRange {
start = date {
year = 2023
month = 7
day = 30
}
endExclusive = date {
year = 2023
month = 7
day = 31
}
}
frequencySpecs +=
SyntheticEventGroupSpecKt.frequencySpec {
frequency = 1

vidRangeSpecs +=
SyntheticEventGroupSpecKt.FrequencySpecKt.vidRangeSpec {
vidRange = populationSpec.vidRange
nonPopulationFieldValues["video_ad.length"] = fieldValue {
durationValue = videoLength
}
}
}
}
}

val testEvents: List<TestEvent> =
SyntheticDataGeneration.generateEvents(TEST_EVENT_DESCRIPTOR, populationSpec, eventGroupSpec)
.map { TestEvent.parseFrom(it.event.toByteString()) }
.toList()

assertThat(testEvents).hasSize(10)
assertThat(testEvents)
.contains(
testEvent {
person = person { gender = Person.Gender.FEMALE }
videoAd = video { length = videoLength }
}
)
}

@Test
fun `sequence from generateEvents throws IllegalArgumentException when vidrange not in subpop`() {
val population = syntheticPopulationSpec {
Expand Down