Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DB-13911][Sizing Calc] Remove ParallelJobs from sizing assessment recommendation as adaptive parallelism now takes care of it. #2017

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

[DB-13911][Sizing Calc] Remove ParallelJobs from sizing assessment recommendation as adaptive parallelism now takes care of it. #2017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d925919
removing ParallelJobs from sizing assessment report
shaharuk-yb Nov 28, 2024
79ac6ab
additional changes
shaharuk-yb Nov 28, 2024
947ac5a
fixing unit tests
shaharuk-yb Nov 28, 2024
9ab2014
Merge branch 'main' into sshaikh/db-13911
shaharuk-yb Dec 2, 2024
a50fa95
removed unnecessary file, rebased with latest main
shaharuk-yb Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions migtests/tests/oracle/assessment-report-test/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 1,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 1
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions ...ts/tests/oracle/bulk-assessment-test/expected_reports/expectedChild1AssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 0,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 0
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions ...ts/tests/oracle/bulk-assessment-test/expected_reports/expectedChild2AssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 1,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 1
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/adventureworks/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 0,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 0
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 1,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 1
},
"FailureReasoning": ""
},
Expand Down
1 change: 0 additions & 1 deletion migtests/tests/pg/mgi/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,6 @@
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 0,
"ParallelVoyagerJobs": 1
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,8 +485,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 1,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 1
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 1,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 1
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/pgtbrus/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 1,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 1
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/rna/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,8 +369,7 @@
"MemoryPerInstance": 32,
"OptimalSelectConnectionsPerNode": 16,
"OptimalInsertConnectionsPerNode": 24,
"EstimatedTimeInMinForImport": 0,
"ParallelVoyagerJobs": 3
"EstimatedTimeInMinForImport": 0
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/sakila/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 0,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 0
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 0,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 0
},
"FailureReasoning": ""
},
Expand Down
3 changes: 1 addition & 2 deletions migtests/tests/pg/stackexchange/expected_files/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,7 @@
"MemoryPerInstance": 16,
"OptimalSelectConnectionsPerNode": 8,
"OptimalInsertConnectionsPerNode": 12,
"EstimatedTimeInMinForImport": 0,
"ParallelVoyagerJobs": 1
"EstimatedTimeInMinForImport": 0
},
"FailureReasoning": ""
},
Expand Down
1 change: 0 additions & 1 deletion yb-voyager/cmd/templates/migration_assessment_report.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,6 @@
<tr><td>Memory per instance(GiB)</td><td>{{ .MemoryPerInstance }}</td></tr>
<tr><td>Optimal select connections per node</td><td>{{ if eq .OptimalSelectConnectionsPerNode 0 }}--{{else}}{{.OptimalSelectConnectionsPerNode }}{{end}}</td></tr>
<tr><td>Optimal insert connections per node</td><td>{{ if eq .OptimalInsertConnectionsPerNode 0 }}--{{else}}{{.OptimalInsertConnectionsPerNode}}{{end}}</td></tr>
<tr><td>Parallel Voyager Jobs</td><td>{{ .ParallelVoyagerJobs }}</td></tr>
<tr><td>Estimated time taken for data import </td><td>{{ .EstimatedTimeInMinForImport }} min</td></tr>
</table>
<h3>Reasoning: </h3>
Expand Down
1 change: 0 additions & 1 deletion yb-voyager/src/migassessment/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ type SizingRecommendation struct {
OptimalSelectConnectionsPerNode int64
OptimalInsertConnectionsPerNode int64
EstimatedTimeInMinForImport float64
ParallelVoyagerJobs float64
shaharuk-yb marked this conversation as resolved.
Show resolved Hide resolved
}

type SizingAssessmentReport struct {
Expand Down
25 changes: 8 additions & 17 deletions yb-voyager/src/migassessment/sizing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ type ExpDataThroughput struct {
type ExpDataLoadTime struct {
csvSizeGB sql.NullFloat64 `db:"csv_size_gb,string"`
migrationTimeSecs sql.NullFloat64 `db:"migration_time_secs,string"`
parallelThreads sql.NullInt64 `db:"parallel_threads,string"`
rowCount sql.NullFloat64 `db:"row_count,string"`
}

Expand All @@ -100,7 +99,6 @@ type IntermediateRecommendation struct {
OptimalSelectConnectionsPerNode int64
OptimalInsertConnectionsPerNode int64
EstimatedTimeInMinForImport float64
ParallelVoyagerJobs float64
FailureReasoning string
}

Expand Down Expand Up @@ -231,7 +229,7 @@ func SizingAssessment() error {
}

// calculate time taken for colocated import
importTimeForColocatedObjects, parallelVoyagerJobsColocated, err := calculateTimeTakenAndParallelJobsForImport(
importTimeForColocatedObjects, err := calculateTimeTakenForImport(
finalSizingRecommendation.ColocatedTables, sourceIndexMetadata, colocatedLoadTimes,
indexImpactOnLoadTimeCommon, columnsImpactOnLoadTimeCommon, COLOCATED)
if err != nil {
Expand All @@ -240,7 +238,7 @@ func SizingAssessment() error {
}

// calculate time taken for sharded import
importTimeForShardedObjects, parallelVoyagerJobsSharded, err := calculateTimeTakenAndParallelJobsForImport(
importTimeForShardedObjects, err := calculateTimeTakenForImport(
finalSizingRecommendation.ShardedTables, sourceIndexMetadata, shardedLoadTimes,
indexImpactOnLoadTimeCommon, columnsImpactOnLoadTimeCommon, SHARDED)
if err != nil {
Expand All @@ -258,7 +256,6 @@ func SizingAssessment() error {
NumNodes: finalSizingRecommendation.NumNodes,
OptimalSelectConnectionsPerNode: finalSizingRecommendation.OptimalSelectConnectionsPerNode,
OptimalInsertConnectionsPerNode: finalSizingRecommendation.OptimalInsertConnectionsPerNode,
ParallelVoyagerJobs: math.Min(float64(parallelVoyagerJobsColocated), float64(parallelVoyagerJobsSharded)),
ColocatedReasoning: reasoning,
EstimatedTimeInMinForImport: importTimeForColocatedObjects + importTimeForShardedObjects,
}
Expand Down Expand Up @@ -368,7 +365,6 @@ func findNumNodesNeededBasedOnThroughputRequirement(sourceIndexMetadata []Source
NumNodes: nodesNeeded,
OptimalSelectConnectionsPerNode: int64(math.Min(float64(previousRecommendation.OptimalSelectConnectionsPerNode), float64(shardedThroughput.selectConnPerNode.Int64))),
OptimalInsertConnectionsPerNode: int64(math.Min(float64(previousRecommendation.OptimalInsertConnectionsPerNode), float64(shardedThroughput.insertConnPerNode.Int64))),
ParallelVoyagerJobs: previousRecommendation.ParallelVoyagerJobs,
ColocatedSize: previousRecommendation.ColocatedSize,
ShardedSize: previousRecommendation.ShardedSize,
EstimatedTimeInMinForImport: previousRecommendation.EstimatedTimeInMinForImport,
Expand Down Expand Up @@ -539,7 +535,6 @@ func checkShardedTableLimit(sourceIndexMetadata []SourceDBMetadata, shardedLimit
NumNodes: previousRecommendation.NumNodes,
OptimalSelectConnectionsPerNode: previousRecommendation.OptimalSelectConnectionsPerNode,
OptimalInsertConnectionsPerNode: previousRecommendation.OptimalInsertConnectionsPerNode,
ParallelVoyagerJobs: previousRecommendation.ParallelVoyagerJobs,
ColocatedSize: 0,
ShardedSize: 0,
EstimatedTimeInMinForImport: previousRecommendation.EstimatedTimeInMinForImport,
Expand Down Expand Up @@ -622,7 +617,6 @@ func shardingBasedOnOperations(sourceIndexMetadata []SourceDBMetadata,
NumNodes: previousRecommendation.NumNodes,
OptimalSelectConnectionsPerNode: colocatedThroughput.selectConnPerNode.Int64,
OptimalInsertConnectionsPerNode: colocatedThroughput.insertConnPerNode.Int64,
ParallelVoyagerJobs: previousRecommendation.ParallelVoyagerJobs,
ColocatedSize: cumulativeColocatedSizeSum,
ShardedSize: cumulativeSizeSharded,
EstimatedTimeInMinForImport: previousRecommendation.EstimatedTimeInMinForImport,
Expand Down Expand Up @@ -702,7 +696,6 @@ func shardingBasedOnTableSizeAndCount(sourceTableMetadata []SourceDBMetadata,
NumNodes: previousRecommendation.NumNodes,
OptimalSelectConnectionsPerNode: previousRecommendation.OptimalSelectConnectionsPerNode,
OptimalInsertConnectionsPerNode: previousRecommendation.OptimalInsertConnectionsPerNode,
ParallelVoyagerJobs: previousRecommendation.ParallelVoyagerJobs,
ColocatedSize: cumulativeColocatedSizeSum,
ShardedSize: cumulativeSizeSharded,
EstimatedTimeInMinForImport: previousRecommendation.EstimatedTimeInMinForImport,
Expand Down Expand Up @@ -881,7 +874,7 @@ func createSizingRecommendationStructure(colocatedLimits []ExpDataColocatedLimit
}

/*
calculateTimeTakenAndParallelJobsForImport estimates the time taken for import of tables.
calculateTimeTakenForImport estimates the time taken for import of tables.
It queries experimental data to find import time estimates for similar object sizes and configurations. For every table
, it tries to find out how much time it would table for importing that table. The function adjusts the
import time on that table by multiplying it by factor based on the indexes. The import time is also converted to
Expand All @@ -897,13 +890,12 @@ Parameters:
Returns:

float64: The estimated time taken for import in minutes.
int64: Total parallel jobs used for import.
error: Error if any
*/
func calculateTimeTakenAndParallelJobsForImport(tables []SourceDBMetadata,
func calculateTimeTakenForImport(tables []SourceDBMetadata,
sourceIndexMetadata []SourceDBMetadata, loadTimes []ExpDataLoadTime,
indexImpactData []ExpDataLoadTimeIndexImpact, numColumnImpactData []ExpDataLoadTimeColumnsImpact,
objectType string) (float64, int64, error) {
objectType string) (float64, error) {
var importTime float64

// we need to calculate the time taken for import for every table.
Expand All @@ -926,7 +918,7 @@ func calculateTimeTakenAndParallelJobsForImport(tables []SourceDBMetadata,
importTime += (loadTimeMultiplicationFactorWrtIndexes * loadTimeMultiplicationFactorWrtNumColumns * tableImportTimeSec) / 60
}

return math.Ceil(importTime), loadTimes[0].parallelThreads.Int64, nil
return math.Ceil(importTime), nil
}

/*
Expand All @@ -945,8 +937,7 @@ Returns:
func getExpDataLoadTime(experimentDB *sql.DB, vCPUPerInstance int, memPerCore int, tableType string) ([]ExpDataLoadTime, error) {
selectQuery := fmt.Sprintf(`
SELECT csv_size_gb,
migration_time_secs,
parallel_threads,
migration_time_secs,
row_count
FROM %v
WHERE num_cores = ?
Expand All @@ -967,7 +958,7 @@ func getExpDataLoadTime(experimentDB *sql.DB, vCPUPerInstance int, memPerCore in
var loadTimes []ExpDataLoadTime
for rows.Next() {
var loadTime ExpDataLoadTime
if err = rows.Scan(&loadTime.csvSizeGB, &loadTime.migrationTimeSecs, &loadTime.parallelThreads, &loadTime.rowCount); err != nil {
if err = rows.Scan(&loadTime.csvSizeGB, &loadTime.migrationTimeSecs, &loadTime.rowCount); err != nil {
return nil, fmt.Errorf("cannot fetch data from experiment data table with query [%s]: %w",
selectQuery, err)
}
Expand Down
Loading