From 5230f5af528bdff7d7f801cd130c8a8e36eacc07 Mon Sep 17 00:00:00 2001 From: Nikhil Chandrappa Date: Mon, 26 Aug 2024 20:36:25 +0000 Subject: [PATCH] [#23630]yugabyted: Modiying the APIs required for the new Migrate Schema Page. Summary: Adding new components for Migrate schema page to display the recommended refactoring graph and suggested modifications. Jira: DB-12545 Test Plan: Manual Testts Reviewers: djiang Reviewed By: djiang Subscribers: yugabyted-dev Differential Revision: https://phorge.dev.yugabyte.com/D37537 --- .../cmd/server/.docs/api/openapi.yaml | 1071 ++++++++++++++++- .../cmd/server/.openapi-generator/FILES | 2 + .../cmd/server/handlers/api_voyager.go | 49 +- .../server/helpers/voyager_migration_utils.go | 9 +- .../model_errors_and_suggestions_details.go | 2 + .../models/model_migrate_schema_task_info.go | 4 + .../models/model_schema_analysis_report.go | 13 + .../model_unsupported_sql_with_details.go | 11 + yugabyted-ui/apiserver/conf/openapi.yml | 80 +- .../conf/openapi/schemas/_index.yaml | 40 + 10 files changed, 1195 insertions(+), 86 deletions(-) create mode 100644 yugabyted-ui/apiserver/cmd/server/models/model_schema_analysis_report.go create mode 100644 yugabyted-ui/apiserver/cmd/server/models/model_unsupported_sql_with_details.go diff --git a/yugabyted-ui/apiserver/cmd/server/.docs/api/openapi.yaml b/yugabyted-ui/apiserver/cmd/server/.docs/api/openapi.yaml index a5ccc14946b4..595f28862a37 100644 --- a/yugabyted-ui/apiserver/cmd/server/.docs/api/openapi.yaml +++ b/yugabyted-ui/apiserver/cmd/server/.docs/api/openapi.yaml @@ -1301,6 +1301,7 @@ components: ErrorsAndSuggestionsDetails: description: Errors and suggestions details example: + issueType: issueType reason: reason GH: GH sqlStatement: sqlStatement @@ -1309,6 +1310,8 @@ components: objectName: objectName objectType: objectType properties: + issueType: + type: string objectType: type: string objectName: @@ -1325,11 +1328,221 @@ components: type: string title: Errors and Suggestions details type: object + UnsupportedSqlWithDetails: + description: Schema for Suggested refactoring tab in Migrate Schema page + example: + suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + properties: + suggestions_errors: + items: + $ref: '#/components/schemas/ErrorsAndSuggestionsDetails' + type: array + unsupported_type: + type: string + count: + type: integer + title: Schema for Suggested refactoring tab in Migrate Schema page + type: object + RefactoringCount: + description: count for automatic and manual refactoring + example: + sql_object_type: sql_object_type + automatic: 1 + manual: 6 + properties: + sql_object_type: + type: string + manual: + type: integer + automatic: + type: integer + title: Count of automatic and manual refactoring + type: object + RecommendedRefactoringGraph: + description: Refectoring recommendations for migrating SQL objects + example: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + properties: + refactor_details: + items: + $ref: '#/components/schemas/RefactoringCount' + type: array + title: Refectoring recommendations per SQL object type + type: object + SchemaAnalysisReport: + description: Voyager data migration metrics details + example: + unsupported_features: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + recommended_refactoring: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + unsupported_datatypes: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + unsupported_functions: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + properties: + unsupported_functions: + items: + $ref: '#/components/schemas/UnsupportedSqlWithDetails' + type: array + unsupported_features: + items: + $ref: '#/components/schemas/UnsupportedSqlWithDetails' + type: array + unsupported_datatypes: + items: + $ref: '#/components/schemas/UnsupportedSqlWithDetails' + type: array + recommended_refactoring: + $ref: '#/components/schemas/RecommendedRefactoringGraph' + title: Schema Analysis report + type: object SqlObjectsDetails: description: Sql obejcts type and count example: - invalidCount: 6 - totalCount: 0 + invalidCount: 5 + totalCount: 5 objectNames: objectNames objectDetails: objectDetails objectType: objectType @@ -1351,14 +1564,16 @@ components: example: import_schema: import_schema suggestions_errors: - - reason: reason + - issueType: issueType + reason: reason GH: GH sqlStatement: sqlStatement filePath: filePath suggestion: suggestion objectName: objectName objectType: objectType - - reason: reason + - issueType: issueType + reason: reason GH: GH sqlStatement: sqlStatement filePath: filePath @@ -1369,17 +1584,394 @@ components: overall_status: overall_status analyze_schema: analyze_schema sql_objects: - - invalidCount: 6 - totalCount: 0 + - invalidCount: 5 + totalCount: 5 objectNames: objectNames objectDetails: objectDetails objectType: objectType - - invalidCount: 6 - totalCount: 0 + - invalidCount: 5 + totalCount: 5 objectNames: objectNames objectDetails: objectDetails objectType: objectType export_schema: export_schema + analysis_history: + - unsupported_features: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + recommended_refactoring: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + unsupported_datatypes: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + unsupported_functions: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - unsupported_features: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + recommended_refactoring: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + unsupported_datatypes: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + unsupported_functions: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + current_analysis_report: + unsupported_features: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + recommended_refactoring: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + unsupported_datatypes: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + unsupported_functions: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 properties: migration_uuid: format: uuid @@ -1396,6 +1988,12 @@ components: items: $ref: '#/components/schemas/ErrorsAndSuggestionsDetails' type: array + current_analysis_report: + $ref: '#/components/schemas/SchemaAnalysisReport' + analysis_history: + items: + $ref: '#/components/schemas/SchemaAnalysisReport' + type: array sql_objects: items: $ref: '#/components/schemas/SqlObjectsDetails' @@ -1587,43 +2185,11 @@ components: $ref: '#/components/schemas/TargetSchemaRecommendations' title: Target YugabyteDB cluster size and schema recommendations type: object - RefactoringCount: - description: count for automatic and manual refactoring - example: - sql_object_type: sql_object_type - automatic: 1 - manual: 1 - properties: - sql_object_type: - type: string - manual: - type: integer - automatic: - type: integer - title: Count of automatic and manual refactoring - type: object - RecommendedRefactoringGraph: - description: Refectoring recommendations for migrating SQL objects - example: - refactor_details: - - sql_object_type: sql_object_type - automatic: 1 - manual: 1 - - sql_object_type: sql_object_type - automatic: 1 - manual: 1 - properties: - refactor_details: - items: - $ref: '#/components/schemas/RefactoringCount' - type: array - title: Refectoring recommendations per SQL object type - type: object UnsupportedSqlInfo: description: Unsupported SQL Info example: unsupported_type: unsupported_type - count: 6 + count: 1 properties: unsupported_type: type: string @@ -1653,17 +2219,17 @@ components: recommendation_summary: recommendation_summary unsupported_features: - unsupported_type: unsupported_type - count: 6 + count: 1 - unsupported_type: unsupported_type - count: 6 + count: 1 recommended_refactoring: refactor_details: - sql_object_type: sql_object_type automatic: 1 - manual: 1 + manual: 6 - sql_object_type: sql_object_type automatic: 1 - manual: 1 + manual: 6 source_environment: total_memory: total_memory total_disk_size: total_disk_size @@ -1671,9 +2237,9 @@ components: total_vcpu: total_vcpu unsupported_functions: - unsupported_type: unsupported_type - count: 6 + count: 1 - unsupported_type: unsupported_type - count: 6 + count: 1 source_database: table_size: 0 total_index_size: 5 @@ -1681,9 +2247,9 @@ components: total_table_size: 1 unsupported_data_types: - unsupported_type: unsupported_type - count: 6 + count: 1 - unsupported_type: unsupported_type - count: 6 + count: 1 properties: summary: $ref: '#/components/schemas/AssessmentReportSummary' @@ -3326,14 +3892,16 @@ components: data: import_schema: import_schema suggestions_errors: - - reason: reason + - issueType: issueType + reason: reason GH: GH sqlStatement: sqlStatement filePath: filePath suggestion: suggestion objectName: objectName objectType: objectType - - reason: reason + - issueType: issueType + reason: reason GH: GH sqlStatement: sqlStatement filePath: filePath @@ -3344,17 +3912,394 @@ components: overall_status: overall_status analyze_schema: analyze_schema sql_objects: - - invalidCount: 6 - totalCount: 0 + - invalidCount: 5 + totalCount: 5 objectNames: objectNames objectDetails: objectDetails objectType: objectType - - invalidCount: 6 - totalCount: 0 + - invalidCount: 5 + totalCount: 5 objectNames: objectNames objectDetails: objectDetails objectType: objectType export_schema: export_schema + analysis_history: + - unsupported_features: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + recommended_refactoring: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + unsupported_datatypes: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + unsupported_functions: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - unsupported_features: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + recommended_refactoring: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + unsupported_datatypes: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + unsupported_functions: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + current_analysis_report: + unsupported_features: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + recommended_refactoring: + refactor_details: + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + - sql_object_type: sql_object_type + automatic: 1 + manual: 6 + unsupported_datatypes: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + unsupported_functions: + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 + - suggestions_errors: + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + - issueType: issueType + reason: reason + GH: GH + sqlStatement: sqlStatement + filePath: filePath + suggestion: suggestion + objectName: objectName + objectType: objectType + unsupported_type: unsupported_type + count: 0 properties: data: $ref: '#/components/schemas/MigrateSchemaTaskInfo' @@ -3382,17 +4327,17 @@ components: recommendation_summary: recommendation_summary unsupported_features: - unsupported_type: unsupported_type - count: 6 + count: 1 - unsupported_type: unsupported_type - count: 6 + count: 1 recommended_refactoring: refactor_details: - sql_object_type: sql_object_type automatic: 1 - manual: 1 + manual: 6 - sql_object_type: sql_object_type automatic: 1 - manual: 1 + manual: 6 source_environment: total_memory: total_memory total_disk_size: total_disk_size @@ -3400,9 +4345,9 @@ components: total_vcpu: total_vcpu unsupported_functions: - unsupported_type: unsupported_type - count: 6 + count: 1 - unsupported_type: unsupported_type - count: 6 + count: 1 source_database: table_size: 0 total_index_size: 5 @@ -3410,9 +4355,9 @@ components: total_table_size: 1 unsupported_data_types: - unsupported_type: unsupported_type - count: 6 + count: 1 - unsupported_type: unsupported_type - count: 6 + count: 1 properties: data: $ref: '#/components/schemas/MigrationAssessmentReport' diff --git a/yugabyted-ui/apiserver/cmd/server/.openapi-generator/FILES b/yugabyted-ui/apiserver/cmd/server/.openapi-generator/FILES index baf44c09bb61..7701b1898ff3 100644 --- a/yugabyted-ui/apiserver/cmd/server/.openapi-generator/FILES +++ b/yugabyted-ui/apiserver/cmd/server/.openapi-generator/FILES @@ -66,6 +66,7 @@ models/model_raft_config.go models/model_recommended_refactoring_graph.go models/model_refactoring_count.go models/model_restore_details.go +models/model_schema_analysis_report.go models/model_slow_query_response_data.go models/model_slow_query_response_schema.go models/model_slow_query_response_ysql_data.go @@ -84,6 +85,7 @@ models/model_target_cluster_spec.go models/model_target_recommendation_item.go models/model_target_schema_recommendations.go models/model_unsupported_sql_info.go +models/model_unsupported_sql_with_details.go models/model_version_info.go models/model_voyager_migrate_data_metrics.go models/model_voyager_migrate_data_metrics_details.go diff --git a/yugabyted-ui/apiserver/cmd/server/handlers/api_voyager.go b/yugabyted-ui/apiserver/cmd/server/handlers/api_voyager.go index 5da0d9eadf10..dd40f93c59ac 100644 --- a/yugabyted-ui/apiserver/cmd/server/handlers/api_voyager.go +++ b/yugabyted-ui/apiserver/cmd/server/handlers/api_voyager.go @@ -363,6 +363,48 @@ func getMigrateSchemaTaskInfoFuture(log logger.Logger, conn *pgxpool.Pool, migra determineStatusOfMigrateSchmeaPhases(log, conn, &schemaPhaseInfoList, &migrateSchemaTaskInfo) + recommendedRefactoringList := []models.RefactoringCount{} + for _, sqlObject := range migrateSchemaTaskInfo.SqlObjects { + var refactorCount models.RefactoringCount + refactorCount.SqlObjectType = sqlObject.ObjectType + refactorCount.Automatic = sqlObject.TotalCount - sqlObject.InvalidCount + refactorCount.Manual = sqlObject.InvalidCount + recommendedRefactoringList = append(recommendedRefactoringList, refactorCount) + } + migrateSchemaTaskInfo.CurrentAnalysisReport.RecommendedRefactoring.RefactorDetails = + recommendedRefactoringList + + conversionIssuesDetailsWithCountMap := map[string]models.UnsupportedSqlWithDetails{} + fmt.Println(migrateSchemaTaskInfo.SuggestionsErrors) + for _, conversionIssue := range migrateSchemaTaskInfo.SuggestionsErrors { + fmt.Println(conversionIssue) + conversionIssuesByType, ok := + conversionIssuesDetailsWithCountMap[conversionIssue.ObjectType] + if ok { + conversionIssuesByType.Count = conversionIssuesByType.Count + 1 + conversionIssuesByType.SuggestionsErrors = append( + conversionIssuesByType.SuggestionsErrors, conversionIssue) + } else { + var newConversionIssuesByType models.UnsupportedSqlWithDetails + newConversionIssuesByType.Count = 1 + newConversionIssuesByType.UnsupportedType = conversionIssue.ObjectType + newConversionIssuesByType.SuggestionsErrors = + append(newConversionIssuesByType.SuggestionsErrors, conversionIssue) + conversionIssuesDetailsWithCountMap[conversionIssue.ObjectType] = + newConversionIssuesByType + } + } + fmt.Println(conversionIssuesDetailsWithCountMap) + + var conversionIssuesDetailsWithCountList []models.UnsupportedSqlWithDetails + for _, value := range conversionIssuesDetailsWithCountMap { + conversionIssuesDetailsWithCountList = append(conversionIssuesDetailsWithCountList, value) + } + fmt.Print(conversionIssuesDetailsWithCountList) + + migrateSchemaTaskInfo.CurrentAnalysisReport.UnsupportedFeatures = + conversionIssuesDetailsWithCountList + MigrateSchemaTaskInfoResponse.Data = migrateSchemaTaskInfo future <- MigrateSchemaTaskInfoResponse } @@ -402,9 +444,12 @@ func determineStatusOfMigrateSchmeaPhases(log logger.Logger, conn *pgxpool.Pool, log.Errorf(migrateSchemaUIDetails.Error.Error()) migrateSchemaTaskInfo.SuggestionsErrors = nil migrateSchemaTaskInfo.SqlObjects = nil + migrateSchemaTaskInfo.AnalyzeSchema = "N/A" + continue } migrateSchemaTaskInfo.SuggestionsErrors = migrateSchemaUIDetails.SuggestionsErrors migrateSchemaTaskInfo.SqlObjects = migrateSchemaUIDetails.SqlObjects + migrateSchemaTaskInfo.AnalyzeSchema = "complete" } else { migrateSchemaTaskInfo.ExportSchema = "in-progress" @@ -747,7 +792,7 @@ func (c *Container) GetVoyagerAssessmentReport(ctx echo.Context) error { for _, unsupportedFeatureType := range assessmentReport.UnsupportedFeatures{ unsupportedFeature := models.UnsupportedSqlInfo{} unsupportedFeature.UnsupportedType = unsupportedFeatureType.FeatureName - unsupportedFeature.Count = int32(len(unsupportedFeatureType.ObjectNames)) + unsupportedFeature.Count = int32(len(unsupportedFeatureType.Objects)) if (unsupportedFeature.Count != 0) { unsupportedFeaturesList = append(unsupportedFeaturesList, unsupportedFeature) } @@ -768,7 +813,7 @@ func getMigrationAssessmentReportFuture(log logger.Logger, migrationUuid string, var assessmentReportPayload string row := conn.QueryRow(context.Background(), RETRIEVE_ASSESSMENT_REPORT, migrationUuid) err := row.Scan(&assessmentReportPayload) - log.Infof(fmt.Sprintf("assessment payload: [%s]", assessmentReportPayload)) + // log.Infof(fmt.Sprintf("assessment payload: [%s]", assessmentReportPayload)) if err != nil { log.Errorf(fmt.Sprintf("[%s] Error while scaning results for query: [%s]", LOGGER_FILE_NAME, "RETRIEVE_ALL_VOYAGER_MIGRATIONS_SQL")) diff --git a/yugabyted-ui/apiserver/cmd/server/helpers/voyager_migration_utils.go b/yugabyted-ui/apiserver/cmd/server/helpers/voyager_migration_utils.go index 68b42518be42..2e010c2b913a 100644 --- a/yugabyted-ui/apiserver/cmd/server/helpers/voyager_migration_utils.go +++ b/yugabyted-ui/apiserver/cmd/server/helpers/voyager_migration_utils.go @@ -117,7 +117,14 @@ type TableColumnsDataTypes struct { type UnsupportedFeature struct { FeatureName string `json:"FeatureName"` - ObjectNames []string `json:"ObjectNames"` + // ObjectNames []string `json:"ObjectNames"` + Objects []ObjectInfo `json:"Objects"` + DocsLink string `json:"DocsLink,omitempty"` +} + +type ObjectInfo struct { + ObjectName string + SqlStatement string } type TableIndexStats struct { diff --git a/yugabyted-ui/apiserver/cmd/server/models/model_errors_and_suggestions_details.go b/yugabyted-ui/apiserver/cmd/server/models/model_errors_and_suggestions_details.go index f50a6e09edf4..81ea04cc5ed5 100644 --- a/yugabyted-ui/apiserver/cmd/server/models/model_errors_and_suggestions_details.go +++ b/yugabyted-ui/apiserver/cmd/server/models/model_errors_and_suggestions_details.go @@ -3,6 +3,8 @@ package models // ErrorsAndSuggestionsDetails - Errors and suggestions details type ErrorsAndSuggestionsDetails struct { + IssueType string `json:"issueType"` + ObjectType string `json:"objectType"` ObjectName string `json:"objectName"` diff --git a/yugabyted-ui/apiserver/cmd/server/models/model_migrate_schema_task_info.go b/yugabyted-ui/apiserver/cmd/server/models/model_migrate_schema_task_info.go index 09b726ac6015..7c0fbdc39c28 100644 --- a/yugabyted-ui/apiserver/cmd/server/models/model_migrate_schema_task_info.go +++ b/yugabyted-ui/apiserver/cmd/server/models/model_migrate_schema_task_info.go @@ -15,5 +15,9 @@ type MigrateSchemaTaskInfo struct { SuggestionsErrors []ErrorsAndSuggestionsDetails `json:"suggestions_errors"` + CurrentAnalysisReport SchemaAnalysisReport `json:"current_analysis_report"` + + AnalysisHistory []SchemaAnalysisReport `json:"analysis_history"` + SqlObjects []SqlObjectsDetails `json:"sql_objects"` } diff --git a/yugabyted-ui/apiserver/cmd/server/models/model_schema_analysis_report.go b/yugabyted-ui/apiserver/cmd/server/models/model_schema_analysis_report.go new file mode 100644 index 000000000000..03ad8d3bc304 --- /dev/null +++ b/yugabyted-ui/apiserver/cmd/server/models/model_schema_analysis_report.go @@ -0,0 +1,13 @@ +package models + +// SchemaAnalysisReport - Voyager data migration metrics details +type SchemaAnalysisReport struct { + + UnsupportedFunctions []UnsupportedSqlWithDetails `json:"unsupported_functions"` + + UnsupportedFeatures []UnsupportedSqlWithDetails `json:"unsupported_features"` + + UnsupportedDatatypes []UnsupportedSqlWithDetails `json:"unsupported_datatypes"` + + RecommendedRefactoring RecommendedRefactoringGraph `json:"recommended_refactoring"` +} diff --git a/yugabyted-ui/apiserver/cmd/server/models/model_unsupported_sql_with_details.go b/yugabyted-ui/apiserver/cmd/server/models/model_unsupported_sql_with_details.go new file mode 100644 index 000000000000..eef2607949c5 --- /dev/null +++ b/yugabyted-ui/apiserver/cmd/server/models/model_unsupported_sql_with_details.go @@ -0,0 +1,11 @@ +package models + +// UnsupportedSqlWithDetails - Schema for Suggested refactoring tab in Migrate Schema page +type UnsupportedSqlWithDetails struct { + + SuggestionsErrors []ErrorsAndSuggestionsDetails `json:"suggestions_errors"` + + UnsupportedType string `json:"unsupported_type"` + + Count int32 `json:"count"` +} diff --git a/yugabyted-ui/apiserver/conf/openapi.yml b/yugabyted-ui/apiserver/conf/openapi.yml index 504affb39161..f6a0db5274f0 100644 --- a/yugabyted-ui/apiserver/conf/openapi.yml +++ b/yugabyted-ui/apiserver/conf/openapi.yml @@ -764,6 +764,8 @@ components: description: Errors and suggestions details type: object properties: + issueType: + type: string objectType: type: string objectName: @@ -778,6 +780,58 @@ components: type: string GH: type: string + UnsupportedSqlWithDetails: + title: Schema for Suggested refactoring tab in Migrate Schema page + description: Schema for Suggested refactoring tab in Migrate Schema page + type: object + properties: + suggestions_errors: + type: array + items: + $ref: '#/components/schemas/ErrorsAndSuggestionsDetails' + unsupported_type: + type: string + count: + type: integer + RefactoringCount: + title: Count of automatic and manual refactoring + description: count for automatic and manual refactoring + type: object + properties: + sql_object_type: + type: string + manual: + type: integer + automatic: + type: integer + RecommendedRefactoringGraph: + title: Refectoring recommendations per SQL object type + description: Refectoring recommendations for migrating SQL objects + type: object + properties: + refactor_details: + type: array + items: + $ref: '#/components/schemas/RefactoringCount' + SchemaAnalysisReport: + title: Schema Analysis report + description: Voyager data migration metrics details + type: object + properties: + unsupported_functions: + type: array + items: + $ref: '#/components/schemas/UnsupportedSqlWithDetails' + unsupported_features: + type: array + items: + $ref: '#/components/schemas/UnsupportedSqlWithDetails' + unsupported_datatypes: + type: array + items: + $ref: '#/components/schemas/UnsupportedSqlWithDetails' + recommended_refactoring: + $ref: '#/components/schemas/RecommendedRefactoringGraph' SqlObjectsDetails: title: Sql Objects description: Sql obejcts type and count @@ -813,6 +867,12 @@ components: type: array items: $ref: '#/components/schemas/ErrorsAndSuggestionsDetails' + current_analysis_report: + $ref: '#/components/schemas/SchemaAnalysisReport' + analysis_history: + type: array + items: + $ref: '#/components/schemas/SchemaAnalysisReport' sql_objects: type: array items: @@ -937,26 +997,6 @@ components: $ref: '#/components/schemas/TargetClusterSpec' target_schema_recommendation: $ref: '#/components/schemas/TargetSchemaRecommendations' - RefactoringCount: - title: Count of automatic and manual refactoring - description: count for automatic and manual refactoring - type: object - properties: - sql_object_type: - type: string - manual: - type: integer - automatic: - type: integer - RecommendedRefactoringGraph: - title: Refectoring recommendations per SQL object type - description: Refectoring recommendations for migrating SQL objects - type: object - properties: - refactor_details: - type: array - items: - $ref: '#/components/schemas/RefactoringCount' UnsupportedSqlInfo: title: Unsupported SQL Info description: Unsupported SQL Info diff --git a/yugabyted-ui/apiserver/conf/openapi/schemas/_index.yaml b/yugabyted-ui/apiserver/conf/openapi/schemas/_index.yaml index 900fc3d17b8a..ce610f57808b 100644 --- a/yugabyted-ui/apiserver/conf/openapi/schemas/_index.yaml +++ b/yugabyted-ui/apiserver/conf/openapi/schemas/_index.yaml @@ -942,15 +942,55 @@ MigrateSchemaTaskInfo: type: array items: $ref: '#/ErrorsAndSuggestionsDetails' + current_analysis_report: + $ref: '#/SchemaAnalysisReport' + analysis_history: + type: array + items: + $ref: '#/SchemaAnalysisReport' sql_objects: type: array items: $ref: '#/SqlObjectsDetails' +SchemaAnalysisReport: + title: Schema Analysis report + description: Voyager data migration metrics details + type: object + properties: + unsupported_functions: + type: array + items: + $ref: '#/UnsupportedSqlWithDetails' + unsupported_features: + type: array + items: + $ref: '#/UnsupportedSqlWithDetails' + unsupported_datatypes: + type: array + items: + $ref: '#/UnsupportedSqlWithDetails' + recommended_refactoring: + $ref: '#/RecommendedRefactoringGraph' +UnsupportedSqlWithDetails: + title: Schema for Suggested refactoring tab in Migrate Schema page + description: Schema for Suggested refactoring tab in Migrate Schema page + type: object + properties: + suggestions_errors: + type: array + items: + $ref: '#/ErrorsAndSuggestionsDetails' + unsupported_type: + type: string + count: + type: integer ErrorsAndSuggestionsDetails: title: Errors and Suggestions details description: Errors and suggestions details type: object properties: + issueType: + type: string objectType: type: string objectName: