[BACKPORT 2024.1][#23630]yugabyted: Modiying the APIs required for th…

…e new Migrate Schema Page.

Summary:
Adding new components for Migrate schema page to display the recommended refactoring graph and suggested modifications.
Jira: DB-12545

Original commit: [[ 5230f5a | 5230f5a ]] / D37537

Test Plan: Manual Testts

Reviewers: djiang

Reviewed By: djiang

Subscribers: yugabyted-dev

Differential Revision: https://phorge.dev.yugabyte.com/D37852

yugabyted-ui/apiserver/cmd/server/.openapi-generator/FILES

@@ -61,6 +61,10 @@ models/model_placement_block.go
 models/model_placement_cloud_info.go
 models/model_placement_info.go
 models/model_raft_config.go
+models/model_recommended_refactoring_graph.go
+models/model_refactoring_count.go
+models/model_restore_details.go
+models/model_schema_analysis_report.go
 models/model_slow_query_response_data.go
 models/model_slow_query_response_schema.go
 models/model_slow_query_response_ysql_data.go
@@ -79,6 +83,7 @@ models/model_target_cluster_spec.go
 models/model_target_recommendation_item.go
 models/model_target_schema_recommendations.go
 models/model_unsupported_sql_info.go
+models/model_unsupported_sql_with_details.go
 models/model_version_info.go
 models/model_voyager_migrate_data_metrics.go
 models/model_voyager_migrate_data_metrics_details.go

yugabyted-ui/apiserver/cmd/server/handlers/api_voyager.go

@@ -363,6 +363,48 @@ func getMigrateSchemaTaskInfoFuture(log logger.Logger, conn *pgxpool.Pool, migra
     determineStatusOfMigrateSchmeaPhases(log, conn, &schemaPhaseInfoList,
         &migrateSchemaTaskInfo)
 
+    recommendedRefactoringList := []models.RefactoringCount{}
+    for _, sqlObject := range migrateSchemaTaskInfo.SqlObjects {
+        var refactorCount models.RefactoringCount
+        refactorCount.SqlObjectType = sqlObject.ObjectType
+        refactorCount.Automatic = sqlObject.TotalCount - sqlObject.InvalidCount
+        refactorCount.Manual = sqlObject.InvalidCount
+        recommendedRefactoringList = append(recommendedRefactoringList, refactorCount)
+    }
+    migrateSchemaTaskInfo.CurrentAnalysisReport.RecommendedRefactoring.RefactorDetails =
+        recommendedRefactoringList
+
+    conversionIssuesDetailsWithCountMap := map[string]models.UnsupportedSqlWithDetails{}
+    fmt.Println(migrateSchemaTaskInfo.SuggestionsErrors)
+    for _, conversionIssue := range migrateSchemaTaskInfo.SuggestionsErrors {
+        fmt.Println(conversionIssue)
+        conversionIssuesByType, ok :=
+            conversionIssuesDetailsWithCountMap[conversionIssue.ObjectType]
+        if ok {
+            conversionIssuesByType.Count = conversionIssuesByType.Count + 1
+            conversionIssuesByType.SuggestionsErrors = append(
+                conversionIssuesByType.SuggestionsErrors, conversionIssue)
+        } else {
+            var newConversionIssuesByType models.UnsupportedSqlWithDetails
+            newConversionIssuesByType.Count = 1
+            newConversionIssuesByType.UnsupportedType = conversionIssue.ObjectType
+            newConversionIssuesByType.SuggestionsErrors =
+                append(newConversionIssuesByType.SuggestionsErrors, conversionIssue)
+            conversionIssuesDetailsWithCountMap[conversionIssue.ObjectType] =
+                    newConversionIssuesByType
+        }
+    }
+    fmt.Println(conversionIssuesDetailsWithCountMap)
+
+    var conversionIssuesDetailsWithCountList []models.UnsupportedSqlWithDetails
+    for _, value := range conversionIssuesDetailsWithCountMap {
+        conversionIssuesDetailsWithCountList = append(conversionIssuesDetailsWithCountList, value)
+    }
+    fmt.Print(conversionIssuesDetailsWithCountList)
+
+    migrateSchemaTaskInfo.CurrentAnalysisReport.UnsupportedFeatures =
+        conversionIssuesDetailsWithCountList
+
     MigrateSchemaTaskInfoResponse.Data = migrateSchemaTaskInfo
     future <- MigrateSchemaTaskInfoResponse
 }
@@ -402,9 +444,12 @@ func determineStatusOfMigrateSchmeaPhases(log logger.Logger, conn *pgxpool.Pool,
                     log.Errorf(migrateSchemaUIDetails.Error.Error())
                     migrateSchemaTaskInfo.SuggestionsErrors = nil
                     migrateSchemaTaskInfo.SqlObjects = nil
+                    migrateSchemaTaskInfo.AnalyzeSchema = "N/A"
+                    continue
                 }
                 migrateSchemaTaskInfo.SuggestionsErrors = migrateSchemaUIDetails.SuggestionsErrors
                 migrateSchemaTaskInfo.SqlObjects = migrateSchemaUIDetails.SqlObjects
+
                 migrateSchemaTaskInfo.AnalyzeSchema = "complete"
             } else {
                 migrateSchemaTaskInfo.ExportSchema = "in-progress"
@@ -747,7 +792,7 @@ func (c *Container) GetVoyagerAssessmentReport(ctx echo.Context) error {
     for _, unsupportedFeatureType := range assessmentReport.UnsupportedFeatures{
         unsupportedFeature := models.UnsupportedSqlInfo{}
         unsupportedFeature.UnsupportedType = unsupportedFeatureType.FeatureName
-        unsupportedFeature.Count = int32(len(unsupportedFeatureType.ObjectNames))
+        unsupportedFeature.Count = int32(len(unsupportedFeatureType.Objects))
         if (unsupportedFeature.Count != 0) {
             unsupportedFeaturesList = append(unsupportedFeaturesList, unsupportedFeature)
         }
@@ -768,7 +813,7 @@ func getMigrationAssessmentReportFuture(log logger.Logger, migrationUuid string,
         var assessmentReportPayload string
         row := conn.QueryRow(context.Background(), RETRIEVE_ASSESSMENT_REPORT, migrationUuid)
         err := row.Scan(&assessmentReportPayload)
-        log.Infof(fmt.Sprintf("assessment payload: [%s]", assessmentReportPayload))
+        // log.Infof(fmt.Sprintf("assessment payload: [%s]", assessmentReportPayload))
         if err != nil {
             log.Errorf(fmt.Sprintf("[%s] Error while scaning results for query: [%s]",
                 LOGGER_FILE_NAME, "RETRIEVE_ALL_VOYAGER_MIGRATIONS_SQL"))

yugabyted-ui/apiserver/cmd/server/helpers/voyager_migration_utils.go

@@ -117,7 +117,14 @@ type TableColumnsDataTypes struct {
 
 type UnsupportedFeature struct {
     FeatureName string   `json:"FeatureName"`
-    ObjectNames []string `json:"ObjectNames"`
+    // ObjectNames []string `json:"ObjectNames"`
+    Objects            []ObjectInfo `json:"Objects"`
+    DocsLink           string       `json:"DocsLink,omitempty"`
+}
+
+type ObjectInfo struct {
+    ObjectName   string
+    SqlStatement string
 }
 
 type TableIndexStats struct {

yugabyted-ui/apiserver/cmd/server/models/model_errors_and_suggestions_details.go

@@ -3,6 +3,8 @@ package models
 // ErrorsAndSuggestionsDetails - Errors and suggestions details
 type ErrorsAndSuggestionsDetails struct {
 
+    IssueType string `json:"issueType"`
+
     ObjectType string `json:"objectType"`
 
     ObjectName string `json:"objectName"`

yugabyted-ui/apiserver/cmd/server/models/model_migrate_schema_task_info.go

@@ -15,5 +15,9 @@ type MigrateSchemaTaskInfo struct {
 
     SuggestionsErrors []ErrorsAndSuggestionsDetails `json:"suggestions_errors"`
 
+    CurrentAnalysisReport SchemaAnalysisReport `json:"current_analysis_report"`
+
+    AnalysisHistory []SchemaAnalysisReport `json:"analysis_history"`
+
     SqlObjects []SqlObjectsDetails `json:"sql_objects"`
 }

yugabyted-ui/apiserver/cmd/server/models/model_schema_analysis_report.go

@@ -0,0 +1,13 @@
+package models
+
+// SchemaAnalysisReport - Voyager data migration metrics details
+type SchemaAnalysisReport struct {
+
+    UnsupportedFunctions []UnsupportedSqlWithDetails `json:"unsupported_functions"`
+
+    UnsupportedFeatures []UnsupportedSqlWithDetails `json:"unsupported_features"`
+
+    UnsupportedDatatypes []UnsupportedSqlWithDetails `json:"unsupported_datatypes"`
+
+    RecommendedRefactoring RecommendedRefactoringGraph `json:"recommended_refactoring"`
+}

yugabyted-ui/apiserver/cmd/server/models/model_unsupported_sql_with_details.go

@@ -0,0 +1,11 @@
+package models
+
+// UnsupportedSqlWithDetails - Schema for Suggested refactoring tab in Migrate Schema page
+type UnsupportedSqlWithDetails struct {
+
+    SuggestionsErrors []ErrorsAndSuggestionsDetails `json:"suggestions_errors"`
+
+    UnsupportedType string `json:"unsupported_type"`
+
+    Count int32 `json:"count"`
+}

yugabyted-ui/apiserver/conf/openapi.yml

@@ -714,6 +714,8 @@ components:
       description: Errors and suggestions details
       type: object
       properties:
+        issueType:
+          type: string
         objectType:
           type: string
         objectName:
@@ -728,6 +730,58 @@ components:
           type: string
         GH:
           type: string
+    UnsupportedSqlWithDetails:
+      title: Schema for Suggested refactoring tab in Migrate Schema page
+      description: Schema for Suggested refactoring tab in Migrate Schema page
+      type: object
+      properties:
+        suggestions_errors:
+          type: array
+          items:
+            $ref: '#/components/schemas/ErrorsAndSuggestionsDetails'
+        unsupported_type:
+          type: string
+        count:
+          type: integer
+    RefactoringCount:
+      title: Count of automatic and manual refactoring
+      description: count for automatic and manual refactoring
+      type: object
+      properties:
+        sql_object_type:
+          type: string
+        manual:
+          type: integer
+        automatic:
+          type: integer
+    RecommendedRefactoringGraph:
+      title: Refectoring recommendations per SQL object type
+      description: Refectoring recommendations for migrating SQL objects
+      type: object
+      properties:
+        refactor_details:
+          type: array
+          items:
+            $ref: '#/components/schemas/RefactoringCount'
+    SchemaAnalysisReport:
+      title: Schema Analysis report
+      description: Voyager data migration metrics details
+      type: object
+      properties:
+        unsupported_functions:
+          type: array
+          items:
+            $ref: '#/components/schemas/UnsupportedSqlWithDetails'
+        unsupported_features:
+          type: array
+          items:
+            $ref: '#/components/schemas/UnsupportedSqlWithDetails'
+        unsupported_datatypes:
+          type: array
+          items:
+            $ref: '#/components/schemas/UnsupportedSqlWithDetails'
+        recommended_refactoring:
+          $ref: '#/components/schemas/RecommendedRefactoringGraph'
     SqlObjectsDetails:
       title: Sql Objects
       description: Sql obejcts type and count
@@ -763,6 +817,12 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/ErrorsAndSuggestionsDetails'
+        current_analysis_report:
+          $ref: '#/components/schemas/SchemaAnalysisReport'
+        analysis_history:
+          type: array
+          items:
+            $ref: '#/components/schemas/SchemaAnalysisReport'
         sql_objects:
           type: array
           items:
@@ -887,26 +947,6 @@ components:
           $ref: '#/components/schemas/TargetClusterSpec'
         target_schema_recommendation:
           $ref: '#/components/schemas/TargetSchemaRecommendations'
-    RefactoringCount:
-      title: Count of automatic and manual refactoring
-      description: count for automatic and manual refactoring
-      type: object
-      properties:
-        sql_object_type:
-          type: string
-        manual:
-          type: integer
-        automatic:
-          type: integer
-    RecommendedRefactoringGraph:
-      title: Refectoring recommendations per SQL object type
-      description: Refectoring recommendations for migrating SQL objects
-      type: object
-      properties:
-        refactor_details:
-          type: array
-          items:
-            $ref: '#/components/schemas/RefactoringCount'
     UnsupportedSqlInfo:
       title: Unsupported SQL Info
       description: Unsupported SQL Info

yugabyted-ui/apiserver/conf/openapi/schemas/_index.yaml

@@ -942,15 +942,55 @@ MigrateSchemaTaskInfo:
       type: array
       items:
         $ref: '#/ErrorsAndSuggestionsDetails'
+    current_analysis_report:
+      $ref: '#/SchemaAnalysisReport'
+    analysis_history:
+      type: array
+      items:
+        $ref: '#/SchemaAnalysisReport'
     sql_objects:
       type: array
       items:
         $ref: '#/SqlObjectsDetails'
+SchemaAnalysisReport:
+  title: Schema Analysis report
+  description: Voyager data migration metrics details
+  type: object
+  properties:
+    unsupported_functions:
+      type: array
+      items:
+        $ref: '#/UnsupportedSqlWithDetails'
+    unsupported_features:
+      type: array
+      items:
+        $ref: '#/UnsupportedSqlWithDetails'
+    unsupported_datatypes:
+      type: array
+      items:
+        $ref: '#/UnsupportedSqlWithDetails'
+    recommended_refactoring:
+      $ref: '#/RecommendedRefactoringGraph'
+UnsupportedSqlWithDetails:
+  title: Schema for Suggested refactoring tab in Migrate Schema page
+  description: Schema for Suggested refactoring tab in Migrate Schema page
+  type: object
+  properties:
+    suggestions_errors:
+      type: array
+      items:
+        $ref: '#/ErrorsAndSuggestionsDetails'
+    unsupported_type:
+      type: string
+    count:
+      type: integer
 ErrorsAndSuggestionsDetails:
   title: Errors and Suggestions details
   description: Errors and suggestions details
   type: object
   properties:
+    issueType:
+      type: string
     objectType:
       type: string
     objectName: