From 6282de42542bea54e1c7ac7c06805b8097983bdf Mon Sep 17 00:00:00 2001 From: Yunkon Kim Date: Thu, 30 May 2024 21:02:50 +0900 Subject: [PATCH] Select the most appropriate VM OS image based on Levenshtein Distance * Add Levenshtein Distance mechanism * Select the keywords representing a server OS info from the source computing infra * Calculate text similarity between keywords and VM OS image ID/Description * Select the VM OS image that shows the highest similarity value --- api/docs.go | 29 +-- api/swagger.json | 29 +-- api/swagger.yaml | 23 +-- pkg/api/rest/common/namespace.go | 12 +- pkg/api/rest/controller/migration.go | 78 ++++---- pkg/api/rest/route/namespace.go | 3 +- pkg/core/common/namespace.go | 2 + pkg/core/recommendation/recommendation.go | 213 +++++++++++++++++++++- 8 files changed, 269 insertions(+), 120 deletions(-) diff --git a/api/docs.go b/api/docs.go index 88b6928..0422211 100644 --- a/api/docs.go +++ b/api/docs.go @@ -72,13 +72,6 @@ const docTemplate = `{ ], "summary": "Migrate an infrastructure on a cloud platform", "parameters": [ - { - "type": "string", - "description": "Namespace ID", - "name": "nsId", - "in": "path", - "required": true - }, { "description": "Specify network, disk, compute, security group, virtual machine, etc.", "name": "InfrastructureInfo", @@ -125,13 +118,6 @@ const docTemplate = `{ ], "summary": "Get the migrated infrastructure on a cloud platform", "parameters": [ - { - "type": "string", - "description": "a namespace ID", - "name": "nsId", - "in": "path", - "required": true - }, { "type": "string", "description": "a infrastructure ID created for migration", @@ -174,13 +160,6 @@ const docTemplate = `{ ], "summary": "Delete the migrated infrastructure on a cloud platform", "parameters": [ - { - "type": "string", - "description": "a namespace ID", - "name": "nsId", - "in": "path", - "required": true - }, { "type": "string", "description": "a infrastructure ID created for migration", @@ -221,7 +200,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "List all namespaces or namespaces' ID", "responses": { @@ -269,7 +248,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "Create namespace", "parameters": [ @@ -315,7 +294,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "Get namespace", "parameters": [ @@ -358,7 +337,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "Delete namespace", "parameters": [ diff --git a/api/swagger.json b/api/swagger.json index 831d510..0694327 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -65,13 +65,6 @@ ], "summary": "Migrate an infrastructure on a cloud platform", "parameters": [ - { - "type": "string", - "description": "Namespace ID", - "name": "nsId", - "in": "path", - "required": true - }, { "description": "Specify network, disk, compute, security group, virtual machine, etc.", "name": "InfrastructureInfo", @@ -118,13 +111,6 @@ ], "summary": "Get the migrated infrastructure on a cloud platform", "parameters": [ - { - "type": "string", - "description": "a namespace ID", - "name": "nsId", - "in": "path", - "required": true - }, { "type": "string", "description": "a infrastructure ID created for migration", @@ -167,13 +153,6 @@ ], "summary": "Delete the migrated infrastructure on a cloud platform", "parameters": [ - { - "type": "string", - "description": "a namespace ID", - "name": "nsId", - "in": "path", - "required": true - }, { "type": "string", "description": "a infrastructure ID created for migration", @@ -214,7 +193,7 @@ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "List all namespaces or namespaces' ID", "responses": { @@ -262,7 +241,7 @@ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "Create namespace", "parameters": [ @@ -308,7 +287,7 @@ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "Get namespace", "parameters": [ @@ -351,7 +330,7 @@ "application/json" ], "tags": [ - "[Namespace] Namespace management" + "[Namespace] Namespace management (To be used)" ], "summary": "Delete namespace", "parameters": [ diff --git a/api/swagger.yaml b/api/swagger.yaml index ce70978..cea8732 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1030,11 +1030,6 @@ paths: - application/json description: It migrates an infrastructure on a cloud platform. parameters: - - description: Namespace ID - in: path - name: nsId - required: true - type: string - description: Specify network, disk, compute, security group, virtual machine, etc. in: body @@ -1066,11 +1061,6 @@ paths: - application/json description: It deletes the migrated infrastructure on a cloud platform. parameters: - - description: a namespace ID - in: path - name: nsId - required: true - type: string - description: a infrastructure ID created for migration in: path name: infraId @@ -1100,11 +1090,6 @@ paths: - application/json description: It gets the migrated infrastructure on a cloud platform. parameters: - - description: a namespace ID - in: path - name: nsId - required: true - type: string - description: a infrastructure ID created for migration in: path name: infraId @@ -1157,7 +1142,7 @@ paths: $ref: '#/definitions/common.SimpleMsg' summary: List all namespaces or namespaces' ID tags: - - '[Namespace] Namespace management' + - '[Namespace] Namespace management (To be used)' post: consumes: - application/json @@ -1186,7 +1171,7 @@ paths: $ref: '#/definitions/common.SimpleMsg' summary: Create namespace tags: - - '[Namespace] Namespace management' + - '[Namespace] Namespace management (To be used)' /ns/{nsId}: delete: consumes: @@ -1212,7 +1197,7 @@ paths: $ref: '#/definitions/common.SimpleMsg' summary: Delete namespace tags: - - '[Namespace] Namespace management' + - '[Namespace] Namespace management (To be used)' get: consumes: - application/json @@ -1241,7 +1226,7 @@ paths: $ref: '#/definitions/common.SimpleMsg' summary: Get namespace tags: - - '[Namespace] Namespace management' + - '[Namespace] Namespace management (To be used)' /readyz: get: consumes: diff --git a/pkg/api/rest/common/namespace.go b/pkg/api/rest/common/namespace.go index b420d32..b0084c3 100644 --- a/pkg/api/rest/common/namespace.go +++ b/pkg/api/rest/common/namespace.go @@ -48,7 +48,7 @@ import ( // // RestDelAllNs godoc // // @Summary Delete all namespaces // // @Description Delete all namespaces -// // @Tags [Namespace] Namespace management +// // @Tags [Namespace] Namespace management (To be used) // // @Accept json // // @Produce json // // @Success 200 {object} common.SimpleMsg @@ -68,7 +68,7 @@ import ( // RestDeleteNs godoc // @Summary Delete namespace // @Description Delete namespace -// @Tags [Namespace] Namespace management +// @Tags [Namespace] Namespace management (To be used) // @Accept json // @Produce json // @Param nsId path string true "Namespace ID" default(ns01) @@ -110,7 +110,7 @@ type JSONResult struct { // RestGetAllNs godoc // @Summary List all namespaces or namespaces' ID // @Description List all namespaces or namespaces' ID -// @Tags [Namespace] Namespace management +// @Tags [Namespace] Namespace management (To be used) // @Accept json // @Produce json // @Success 200 {object} JSONResult{[DEFAULT]=RestGetAllNsResponse,[ID]=common.IdList} "Different return structures by the given option param" @@ -137,7 +137,7 @@ func RestGetAllNs(c echo.Context) error { // RestGetNs godoc // @Summary Get namespace // @Description Get namespace -// @Tags [Namespace] Namespace management +// @Tags [Namespace] Namespace management (To be used) // @Accept json // @Produce json // @Param nsId path string true "Namespace ID" default(ns01) @@ -172,7 +172,7 @@ func RestGetNs(c echo.Context) error { // RestPostNs godoc // @Summary Create namespace // @Description Create namespace -// @Tags [Namespace] Namespace management +// @Tags [Namespace] Namespace management (To be used) // @Accept json // @Produce json // @Param nsReq body common.NsReq true "Details for a new namespace" @@ -205,7 +205,7 @@ func RestPostNs(c echo.Context) error { // // RestPutNs godoc // // @Summary Update namespace // // @Description Update namespace -// // @Tags [Namespace] Namespace management +// // @Tags [Namespace] Namespace management (To be used) // // @Accept json // // @Produce json // // @Param nsId path string true "Namespace ID" default(ns01) diff --git a/pkg/api/rest/controller/migration.go b/pkg/api/rest/controller/migration.go index 996dcec..96a4abf 100644 --- a/pkg/api/rest/controller/migration.go +++ b/pkg/api/rest/controller/migration.go @@ -46,7 +46,6 @@ type MigrateInfraResponse struct { // @Tags [Migration] Infrastructure // @Accept json // @Produce json -// @Param nsId path string true "Namespace ID" // @Param InfrastructureInfo body MigrateInfraRequest true "Specify network, disk, compute, security group, virtual machine, etc." // @Success 200 {object} MigrateInfraResponse "Successfully migrated infrastructure on a cloud platform" // @Failure 404 {object} model.Response @@ -55,16 +54,17 @@ type MigrateInfraResponse struct { func MigrateInfra(c echo.Context) error { // [Note] Input section - nsId := c.Param("nsId") - if nsId == "" { - err := fmt.Errorf("invalid request, namespace ID (nsId: %s) is required", nsId) - log.Warn().Msg(err.Error()) - res := model.Response{ - Success: false, - Text: err.Error(), - } - return c.JSON(http.StatusBadRequest, res) - } + // nsId := c.Param("nsId") + // if nsId == "" { + // err := fmt.Errorf("invalid request, namespace ID (nsId: %s) is required", nsId) + // log.Warn().Msg(err.Error()) + // res := model.Response{ + // Success: false, + // Text: err.Error(), + // } + // return c.JSON(http.StatusBadRequest, res) + // } + nsId := common.DefaulNamespaceId req := &MigrateInfraRequest{} if err := c.Bind(req); err != nil { @@ -76,12 +76,14 @@ func MigrateInfra(c echo.Context) error { nsInfo, err := common.GetNamespace(nsId) if err != nil { - log.Error().Err(err).Msg("failed to get the namespace") - res := model.Response{ - Success: false, - Text: err.Error(), + // [temporary code block] Create a namespace as a default + log.Warn().Msgf("failed to get the namespace (nsId: %s)", nsId) + log.Info().Msg("create a namespace as a default (nsId: ns-mig01)") + nsReq := common.NsReq{ + Name: common.DefaulNamespaceId, } - return c.JSON(http.StatusInternalServerError, res) + nsInfo, err = common.CreateNamespace(nsReq) + } if nsInfo.Id == "" { @@ -121,7 +123,6 @@ func MigrateInfra(c echo.Context) error { // @Tags [Migration] Infrastructure // @Accept json // @Produce json -// @Param nsId path string true "a namespace ID" // @Param infraId path string true "a infrastructure ID created for migration" // @Success 200 {object} MigrateInfraResponse "Successfully got the migrated infrastructure on a cloud platform" // @Failure 404 {object} model.Response @@ -130,16 +131,17 @@ func MigrateInfra(c echo.Context) error { func GetInfra(c echo.Context) error { // [Note] Input section - nsId := c.Param("vsId") - if nsId == "" { - err := fmt.Errorf("invalid request, the nanespace ID (nsId: %s) is required", nsId) - log.Warn().Msg(err.Error()) - res := model.Response{ - Success: false, - Text: err.Error(), - } - return c.JSON(http.StatusBadRequest, res) - } + // nsId := c.Param("nsId") + // if nsId == "" { + // err := fmt.Errorf("invalid request, the nanespace ID (nsId: %s) is required", nsId) + // log.Warn().Msg(err.Error()) + // res := model.Response{ + // Success: false, + // Text: err.Error(), + // } + // return c.JSON(http.StatusBadRequest, res) + // } + nsId := common.DefaulNamespaceId infraId := c.Param("infraId") if infraId == "" { @@ -173,7 +175,6 @@ func GetInfra(c echo.Context) error { // @Tags [Migration] Infrastructure // @Accept json // @Produce json -// @Param nsId path string true "a namespace ID" // @Param infraId path string true "a infrastructure ID created for migration" // @Success 200 {object} model.Response "Successfully deleted the migrated infrastructure on a cloud platform" // @Failure 404 {object} model.Response @@ -182,16 +183,17 @@ func GetInfra(c echo.Context) error { func DeleteInfra(c echo.Context) error { // [Note] Input section - nsId := c.Param("vsId") - if nsId == "" { - err := fmt.Errorf("invalid request, the nanespace ID (nsId: %s) is required", nsId) - log.Warn().Msg(err.Error()) - res := model.Response{ - Success: false, - Text: err.Error(), - } - return c.JSON(http.StatusBadRequest, res) - } + // nsId := c.Param("nsId") + // if nsId == "" { + // err := fmt.Errorf("invalid request, the nanespace ID (nsId: %s) is required", nsId) + // log.Warn().Msg(err.Error()) + // res := model.Response{ + // Success: false, + // Text: err.Error(), + // } + // return c.JSON(http.StatusBadRequest, res) + // } + nsId := common.DefaulNamespaceId infraId := c.Param("infraId") if infraId == "" { diff --git a/pkg/api/rest/route/namespace.go b/pkg/api/rest/route/namespace.go index b7308c3..92bb97a 100644 --- a/pkg/api/rest/route/namespace.go +++ b/pkg/api/rest/route/namespace.go @@ -1,8 +1,9 @@ package route import ( - "github.com/cloud-barista/cm-beetle/pkg/api/rest/common" "github.com/labstack/echo/v4" + + "github.com/cloud-barista/cm-beetle/pkg/api/rest/common" ) // /beetle/ns/* diff --git a/pkg/core/common/namespace.go b/pkg/core/common/namespace.go index 87a01f4..ca83510 100644 --- a/pkg/core/common/namespace.go +++ b/pkg/core/common/namespace.go @@ -24,6 +24,8 @@ import ( //"github.com/cloud-barista/cm-beetle/src/core/mcis" ) +var DefaulNamespaceId = "mig-ns01" + type NsReq struct { Name string `json:"name" example:"ns01"` Description string `json:"description" example:"Description for this namespace"` diff --git a/pkg/core/recommendation/recommendation.go b/pkg/core/recommendation/recommendation.go index 67a19fe..8c7966c 100644 --- a/pkg/core/recommendation/recommendation.go +++ b/pkg/core/recommendation/recommendation.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "regexp" "strings" "github.com/cloud-barista/cb-tumblebug/src/core/mcir" @@ -213,10 +214,11 @@ func Recommend(srcInfra []infra.Infra) (cloudmodel.InfraMigrationReq, error) { log.Debug().Msgf("select the 1st recommended virtual machine: %+v", resRecommVmList[0]) recommendedSpec := resRecommVmList[0].Id - name := fmt.Sprintf("rehosted-%s-%s", server.Compute.OS.Node.Hostname, server.Compute.OS.Node.Machineid) + // name := fmt.Sprintf("rehosted-%s-%s", server.Compute.OS.Node.Hostname, server.Compute.OS.Node.Machineid) + name := fmt.Sprintf("rehosted-%s", server.Compute.OS.Node.Hostname) //////////////////////////////////////// - // Search and set target OS image (e.g. ubuntu22.04) + // Search and set target VM image (e.g. ubuntu22.04) method = "POST" url = fmt.Sprintf("%s/mcisDynamicCheckRequest", epTumblebug) @@ -245,15 +247,28 @@ func Recommend(srcInfra []infra.Infra) (cloudmodel.InfraMigrationReq, error) { log.Trace().Msgf("resMcisDynamicCheck: %+v", resMcisDynamicCheck) - // candidateImages := resMcisDynamicCheck.ReqCheck[0].Image + if len(resMcisDynamicCheck.ReqCheck) == 0 { + log.Warn().Msg("no VM OS image recommended for the inserted PM/VM") + continue + } - // [TBD] Select VM image by TextSimilarity + keywords := fmt.Sprintf("%s %s %s %s", + server.Compute.OS.OS.Vendor, + server.Compute.OS.OS.Version, + server.Compute.OS.OS.Architecture, + server.Compute.ComputeResource.RootDisk.Type) + log.Debug().Msg("keywords for the VM OS image recommendation: " + keywords) - image := fmt.Sprintf("%s+%s+%s", providerName, regionName, osNameWithVersion) + // Select VM OS image via LevenshteinDistance-based text similarity + delimiters1 := []string{" ", "-", "_", ",", "(", ")", "[", "]"} + delimiters2 := delimiters1 + vmOsImageId := FindBestVmOsImage(keywords, delimiters1, resMcisDynamicCheck.ReqCheck[0].Image, delimiters2) + + // vmOsImage := fmt.Sprintf("%s+%s+%s", providerName, regionName, osNameWithVersion) vm := cloudmodel.HostMigrationReq{ ConnectionName: "", - CommonImage: image, + CommonImage: vmOsImageId, CommonSpec: recommendedSpec, Description: "a recommended virtual machine", Label: "rehosted", @@ -278,3 +293,189 @@ func MBtoGiB(mb float64) uint32 { gib := (mb * bytesInMB) / bytesInGiB return uint32(gib) } + +// FindBestVmOsImage finds the best matching image based on the similarity scores +func FindBestVmOsImage(keywords string, kwDelimiters []string, vmImages []mcir.TbImageInfo, imgDelimiters []string) string { + + var bestVmOsImageID string + var highestScore float64 + + for _, image := range vmImages { + score := calculateSimilarity(keywords, kwDelimiters, image.CspImageName, imgDelimiters) + if score > highestScore { + highestScore = score + bestVmOsImageID = image.Id + } + log.Trace().Msgf("VmImageName: %s, score: %f", image.CspImageName, score) + + } + log.Debug().Msgf("bestVmOsImageID: %s, highestScore: %f", bestVmOsImageID, highestScore) + + return bestVmOsImageID +} + +// calculateSimilarity calculates the similarity between two texts based on word similarities +func calculateSimilarity(text1 string, delimiters1 []string, text2 string, delimiters2 []string) float64 { + words1 := splitToArray(text1, delimiters1) + words2 := splitToArray(text2, delimiters2) + + // Calculate the similarity between two texts based on word similarities + totalSimilarity := 0.0 + for _, word1 := range words1 { + bestMatch := 0.0 + for _, word2 := range words2 { + similarity := wordSimilarity(word1, word2) + if similarity > bestMatch { + bestMatch = similarity + } + + totalSimilarity += activateByReLU(bestMatch, 0.3) + } + } + + // Normalize by the number of words + return totalSimilarity / float64(len(words1)) +} + +func splitToArray(text string, delimiters []string) []string { + + if len(delimiters) == 0 { + log.Warn().Msg("warning: delimiters empty. delimiters are empty. Using space (' ') as default delimiter.") + delimiters = []string{" "} + } + + // Convert to lowercase + text = strings.ToLower(text) + + // Create a regular expression pattern for the delimiters + pattern := strings.Join(delimiters, "|") + re := regexp.MustCompile(pattern) + + // Split text by the delimiters + arr := re.Split(text, -1) + + return arr +} + +// wordSimilarity calculates the similarity between two words based on Levenshtein distance +func wordSimilarity(word1, word2 string) float64 { + maxLen := float64(max(len(word1), len(word2))) + if maxLen == 0 { + return 1.0 + } + return 1.0 - float64(LevenshteinDistance(word1, word2))/maxLen +} + +// activateByReLU applies a ReLU function that activates if the similarity is greater than a threshold +func activateByReLU(similarity, threshold float64) float64 { + if similarity > threshold { + return similarity + } + return 0.0 +} + +// max returns the maximum of two integers +func max(a, b int) int { + if a > b { + return a + } + return b +} + +// LevenshteinDistance calculates the Levenshtein distance between two strings +func LevenshteinDistance(text1, text2 string) int { + text1Len, text2Len := len(text1), len(text2) + if text1Len == 0 { + return text2Len + } + if text2Len == 0 { + return text1Len + } + matrix := make([][]int, text1Len+1) + for i := range matrix { + matrix[i] = make([]int, text2Len+1) + } + for i := 0; i <= text1Len; i++ { + matrix[i][0] = i + } + for j := 0; j <= text2Len; j++ { + matrix[0][j] = j + } + for i := 1; i <= text1Len; i++ { + for j := 1; j <= text2Len; j++ { + cost := 0 + if text1[i-1] != text2[j-1] { + cost = 1 + } + matrix[i][j] = min(matrix[i-1][j]+1, min(matrix[i][j-1]+1, matrix[i-1][j-1]+cost)) + } + } + return matrix[text1Len][text2Len] +} + +// min returns the minimum of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// // JaccardSimilarity calculates the Jaccard similarity between two strings +// func JaccardSimilarity(text1, delimiter1, text2, delimiter2 string) float64 { + +// // Convert a string into a set of words (e.g., "hello world" -> {"hello", "world"}) +// setA := toSet(text1, delimiter1) +// setB := toSet(text2, delimiter2) + +// // Calculate the Jaccard similarity +// intersectionSize := len(intersection(setA, setB)) +// unionSize := len(union(setA, setB)) + +// if unionSize == 0 { +// return 0 +// } + +// return float64(intersectionSize) / float64(unionSize) +// } + +// func intersection(setA, setB map[string]struct{}) map[string]struct{} { +// intersection := make(map[string]struct{}) +// for item := range setA { +// if _, found := setB[item]; found { +// intersection[item] = struct{}{} +// } +// } +// return intersection +// } + +// func union(setA, setB map[string]struct{}) map[string]struct{} { +// union := make(map[string]struct{}) +// for item := range setA { +// union[item] = struct{}{} +// } +// for item := range setB { +// union[item] = struct{}{} +// } +// return union +// } + +// func toSet(text, delimiter string) map[string]struct{} { + +// if delimiter == "" { +// log.Warn().Msg("delimiter is empty. Set it to a space (' ')") +// delimiter = " " +// } + +// // Convert to lowercase +// text = strings.ToLower(text) + +// // Split text by delimiter +// arr := strings.Split(text, delimiter) + +// set := make(map[string]struct{}) +// for _, item := range arr { +// set[item] = struct{}{} +// } +// return set +// }