Skip to content

Commit

Permalink
Merge pull request #42 from buildbarn/db_perf
Browse files Browse the repository at this point in the history 
optionally save more or less data to the database to improve overall performance.
  • Loading branch information
@trey-ivy
trey-ivy authored Oct 18, 2024
2 parents 915c691 + 5eca64b commit be28d3c
Show file tree
Hide file tree
Showing 8 changed files with 119 additions and 78 deletions.
72 changes: 40 additions & 32 deletions pkg/processing/save.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,23 +239,8 @@ func (act SaveActor) saveTargetCompletion(ctx context.Context, targetCompletion
Save(ctx)
}

func (act SaveActor) saveTargetPair(ctx context.Context, targetPair summary.TargetPair, label string) (*ent.TargetPair, error) {
configuration := targetPair.Configuration
completion := targetPair.Completion

targetConfiguration, err := act.saveTargetConfiguration(ctx, configuration)
if err != nil {
return nil, err
}

targetCompletion, err := act.saveTargetCompletion(ctx, completion)
if err != nil {
return nil, err
}

func (act SaveActor) saveTargetPair(ctx context.Context, targetPair summary.TargetPair, label string, enrich bool) (*ent.TargetPair, error) {
create := act.db.TargetPair.Create().
SetCompletion(targetCompletion).
SetConfiguration(targetConfiguration).
SetLabel(label).
SetDurationInMs(targetPair.DurationInMs).
SetSuccess(targetPair.Success).
Expand All @@ -267,16 +252,35 @@ func (act SaveActor) saveTargetPair(ctx context.Context, targetPair summary.Targ
create = create.SetAbortReason(reason)
}

if enrich {
configuration := targetPair.Configuration
completion := targetPair.Completion

targetConfiguration, err := act.saveTargetConfiguration(ctx, configuration)
if err != nil {
return nil, err
}

targetCompletion, err := act.saveTargetCompletion(ctx, completion)
if err != nil {
return nil, err
}
create.SetCompletion(targetCompletion).SetConfiguration(targetConfiguration)
}

return create.Save(ctx)
}

// TODO: is there a more effiient way to do bulk updates instead of sequentially adding everything to the database one object at a time?
// ironically, MapBulkCreate doesn't work for the map(string)TargetPair. Its expecting an int index, not a label.
func (act SaveActor) saveTargets(ctx context.Context, summary *summary.Summary) ([]*ent.TargetPair, error) {
if summary.SkipTargetData {
return []*ent.TargetPair{}, nil
}
var result []*ent.TargetPair = make([]*ent.TargetPair, len(summary.Targets))
i := 0
for label, pair := range summary.Targets {
targetPair, err := act.saveTargetPair(ctx, pair, label)
targetPair, err := act.saveTargetPair(ctx, pair, label, summary.EnrichTargetData)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -375,33 +379,37 @@ func (act SaveActor) saveTestResults(ctx context.Context, testResults []summary.
}).Save(ctx)
}

func (act SaveActor) saveTestCollection(ctx context.Context, testCollection summary.TestsCollection, label string) (*ent.TestCollection, error) {
testSummary, err := act.saveTestSummary(ctx, testCollection.TestSummary, label)
if err != nil {
return nil, err
}
testResults, err := act.saveTestResults(ctx, testCollection.TestResults)
if err != nil {
return nil, err
}
return act.db.TestCollection.Create().
func (act SaveActor) saveTestCollection(ctx context.Context, testCollection summary.TestsCollection, label string, encrich bool) (*ent.TestCollection, error) {
create := act.db.TestCollection.Create().
SetLabel(label).
SetTestSummary(testSummary).
AddTestResults(testResults...).
SetOverallStatus(testcollection.OverallStatus(testCollection.OverallStatus.String())).
SetStrategy(testCollection.Strategy).
SetCachedLocally(testCollection.CachedLocally).
SetCachedRemotely(testCollection.CachedRemotely).
SetDurationMs(testCollection.DurationMs).
SetFirstSeen((testCollection.FirstSeen)).
Save(ctx)
SetFirstSeen((testCollection.FirstSeen))

if encrich {

testSummary, err := act.saveTestSummary(ctx, testCollection.TestSummary, label)
if err != nil {
return nil, err
}
testResults, err := act.saveTestResults(ctx, testCollection.TestResults)
if err != nil {
return nil, err
}
create.SetTestSummary(testSummary).AddTestResults(testResults...)
}

return create.Save(ctx)
}

func (act SaveActor) saveTests(ctx context.Context, summary *summary.Summary) ([]*ent.TestCollection, error) {
var result []*ent.TestCollection = make([]*ent.TestCollection, len(summary.Tests))
i := 0
for label, collection := range summary.Tests {
testCollection, err := act.saveTestCollection(ctx, collection, label)
testCollection, err := act.saveTestCollection(ctx, collection, label, summary.EnrichTargetData)
if err != nil {
return nil, err
}
Expand Down
37 changes: 29 additions & 8 deletions pkg/summary/summarizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,10 +184,11 @@ func (s Summarizer) handleBuildConfiguration(configuration *bes.Configuration) {
// handleTargetConfigured
func (s Summarizer) handleTargetConfigured(target *bes.TargetConfigured, label string, timestamp time.Time) {
if len(label) == 0 {
panic("missing a target label for target configured event!")
slog.Warn("missing a target label for target configured event!")
return
}
if target == nil {
slog.Debug(fmt.Sprintf("missing target for label %s on targetConfigured", label))
slog.Warn(fmt.Sprintf("missing target for label %s on targetConfigured", label))
return
}

Expand All @@ -213,18 +214,21 @@ func (s Summarizer) handleTargetConfigured(target *bes.TargetConfigured, label s
// handleTargetCompleted
func (s Summarizer) handleTargetCompleted(target *bes.TargetComplete, label string, aborted *bes.Aborted, timestamp time.Time) {
if len(label) == 0 {
panic("label is empty for a target completed event")
slog.Error("label is empty for a target completed event")
return
}

if s.summary.Targets == nil {
panic(fmt.Sprintf("target completed event received before any target configured messages for label %s,", label))
slog.Warn(fmt.Sprintf("target completed event received before any target configured messages for label %s,", label))
return
}

var targetPair TargetPair
targetPair, ok := s.summary.Targets[label]

if !ok {
panic(fmt.Sprintf("target completed event received for label %s before target configured message received", label))
slog.Warn(fmt.Sprintf("target completed event received for label %s before target configured message received", label))
return
}

var targetCompletion TargetComplete
Expand Down Expand Up @@ -359,17 +363,20 @@ func processExecutionInfo(testResult *bes.TestResult) ExecutionInfo {
// handleTestSummary
func (s Summarizer) handleTestSummary(testSummary *bes.TestSummary, label string) {
if len(label) == 0 {
panic("missing label on handleTestSummary event")
slog.Error("missing label on handleTestSummary event")
return
}

if testSummary == nil {
panic(fmt.Sprintf("missing test summary object for handleTestSummary event for label %s", label))
slog.Warn("missing test summary object for handleTestSummary event for label %s", label, nil)
return
}

testCollection, ok := s.summary.Tests[label]

if !ok {
panic(fmt.Sprintf("received a test summary event but never first saw a test result for label %s", label))
slog.Warn("received a test summary event but never first saw a test result for label %s", label, nil)
return
}

tSummary := testCollection.TestSummary
Expand Down Expand Up @@ -787,6 +794,20 @@ func (s Summarizer) handleStructuredCommandLine(structuredCommandLine *bescore.C
s.summary.BuildURL = s.summary.InvocationSummary.EnvVars["BUILD_URL"]
s.summary.BuildUUID = uuid.NewSHA1(uuid.NameSpaceURL, []byte(s.summary.BuildURL))

// Set SkipTargetData
if skipTargetSaveEnvVarVal, ok := s.summary.EnvVars["BB_PORTAL_SKIP_SAVE_TARGETS"]; ok {
if skipTargetSaveEnvVarVal == "TRUE" {
s.summary.SkipTargetData = true
}
}

// Set EnrichTargetData
if enrichTargetDataVal, ok := s.summary.EnvVars["BB_PORTAL_ENRICH_TARGET_DATA"]; ok {
if enrichTargetDataVal == "TRUE" {
s.summary.EnrichTargetData = true
}
}

return nil
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/summary/summary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ type Summary struct {
PlatformName string
ProfileName string
ConfigrationMnemonic string
SkipTargetData bool
EnrichTargetData bool
}

// Metrics holds Build metrics details
Expand Down
26 changes: 14 additions & 12 deletions pkg/summary/testdata/snapshots/nextjs_build.bep.ndjson.golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.579117733Z"
"FirstSeen": "2024-10-18T18:21:46.723807658Z"
},
"//next.js:build_smoke_test": {
"TestSummary": {
Expand Down Expand Up @@ -315,7 +315,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.578731697Z"
"FirstSeen": "2024-10-18T18:21:46.723426532Z"
},
"//next.js:build_test": {
"TestSummary": {
Expand Down Expand Up @@ -376,7 +376,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.578686265Z"
"FirstSeen": "2024-10-18T18:21:46.72338211Z"
},
"//react-webpack:build_smoke_test": {
"TestSummary": {
Expand Down Expand Up @@ -437,7 +437,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.575992687Z"
"FirstSeen": "2024-10-18T18:21:46.721110312Z"
},
"//react/src:lint": {
"TestSummary": {
Expand Down Expand Up @@ -498,7 +498,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.575237584Z"
"FirstSeen": "2024-10-18T18:21:46.720344848Z"
},
"//react/src:src_typecheck_test": {
"TestSummary": {
Expand Down Expand Up @@ -559,7 +559,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.573621233Z"
"FirstSeen": "2024-10-18T18:21:46.718683677Z"
},
"//react/src:test": {
"TestSummary": {
Expand Down Expand Up @@ -620,7 +620,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.575515986Z"
"FirstSeen": "2024-10-18T18:21:46.720660573Z"
},
"//react/src:test_lib_typecheck_test": {
"TestSummary": {
Expand Down Expand Up @@ -681,7 +681,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.574122685Z"
"FirstSeen": "2024-10-18T18:21:46.719258972Z"
},
"//react:build_smoke_test": {
"TestSummary": {
Expand Down Expand Up @@ -742,7 +742,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.57446967Z"
"FirstSeen": "2024-10-18T18:21:46.719595516Z"
},
"//vue:build_test": {
"TestSummary": {
Expand Down Expand Up @@ -803,7 +803,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.574615247Z"
"FirstSeen": "2024-10-18T18:21:46.719726672Z"
},
"//vue:type-check": {
"TestSummary": {
Expand Down Expand Up @@ -864,7 +864,7 @@
"CachedLocally": true,
"CachedRemotely": false,
"DurationMs": 0,
"FirstSeen": "2024-10-10T18:40:35.575000133Z"
"FirstSeen": "2024-10-18T18:21:46.720041905Z"
}
},
"Targets": {
Expand Down Expand Up @@ -3023,6 +3023,8 @@
"CPU": "darwin_arm64",
"PlatformName": "darwin_arm64",
"ProfileName": "command.profile.gz",
"ConfigrationMnemonic": "darwin_arm64-fastbuild"
"ConfigrationMnemonic": "darwin_arm64-fastbuild",
"SkipTargetData": false,
"EnrichTargetData": false
}
}
Loading

0 comments on commit be28d3c

Please sign in to comment.