diff --git a/assets/cloudimage.csv b/assets/cloudimage.csv index 0516d73b1..b8379ae00 100644 --- a/assets/cloudimage.csv +++ b/assets/cloudimage.csv @@ -92,8 +92,8 @@ GCP,all,https://www.googleapis.com/compute/v1/projects/debian-cloud/global/debia GCP,all,https://www.googleapis.com/compute/v1/projects/deeplearning-platform-release/global/images/tf-2-15-cu121-v20240417-debian-11,Debian 11,"Google, Deep Learning VM for TensorFlow 2.15 with CUDA 12.1, M120, Debian 11, Python 3.10, with TensorFlow 2.15 for CUDA 12.1 with Intel MKL-DNN preinstalled",,vm GCP,all,https://www.googleapis.com/compute/v1/projects/windows-cloud/global/images/windows-server-2012-r2-dc-v20221014,Windows Server 2012 R2,,,vm ALIBABA,all,ubuntu_18_04_x64_20G_alibase_20240223.vhd,Ubuntu 18.04,,,vm -ALIBABA,all,ubuntu_20_04_x64_20G_alibase_20240530.vhd,Ubuntu 20.04,,,vm -ALIBABA,all,ubuntu_22_04_x64_20G_alibase_20240628.vhd,Ubuntu 22.04,,,vm +ALIBABA,all,ubuntu_20_04_x64_20G_alibase_20240723.vhd,Ubuntu 20.04,,,vm +ALIBABA,all,ubuntu_22_04_x64_20G_alibase_20240710.vhd,Ubuntu 22.04,,,vm ALIBABA,all,aliyun_3_x64_20G_alibase_20240528.vhd,Alibaba Cloud Linux 3.2104,,,vm|k8s TENCENT,all,img-pi0ii46r,Ubuntu 18.04,,,vm|k8s TENCENT,all,img-22trbn9x,Ubuntu 20.04,,,vm|k8s @@ -102,9 +102,11 @@ TENCENT,all,img-bpsjtw7n,Windows Server 2012 R2,,,vm IBM,us-south,r006-9de77234-3189-42f8-982d-f2266477cfe0,Ubuntu 18.04,,,vm IBM,us-south,r006-b4d8d8d4-7768-4a1b-9434-9483ca821a0d,Windows Server 2012 R2,,,vm IBM,br-sao,r042-92d1cd12-f014-4b9a-abf8-c5ca6494a9e5,Ubuntu 18.04,,,vm +IBM,br-sao,r042-d932aff6-577b-40c9-b1c0-59820b847089,Ubuntu 22.04,Ubuntu Linux 22.04 Jammy Jellyfish Minimal Install (s390x),,vm IBM,ca-tor,r038-e92647cf-8be9-438a-b94c-251cc86bc99a,Ubuntu 18.04,,,vm IBM,us-east,r014-dc446598-a1b5-41c3-a1d6-add3afaf264e,Ubuntu 18.04,,,vm IBM,eu-de,r010-1f68eb2d-f35c-4959-8f4b-2b2f9cf78102,Ubuntu 18.04,,,vm +IBM,eu-de,r010-450355ae-7f70-48fe-9aae-7cf4ac82bc9c,Ubuntu 22.04,Ubuntu Linux 22.04 LTS Jammy Jellyfish Minimal Install (amd64),,vm IBM,eu-gb,r018-1d7417c6-893e-49d4-b14d-9643d6b29812,Ubuntu 18.04,,,vm IBM,jp-osa,r034-522c639c-52e1-4cab-8dfb-bc0fb9f6f577,Ubuntu 18.04,,,vm IBM,au-syd,r026-a8c25ce6-0ca1-43e9-9b41-411c6217b8b8,Ubuntu 18.04,,,vm diff --git a/init/init.py b/init/init.py index ca529729c..6f1836930 100755 --- a/init/init.py +++ b/init/init.py @@ -33,7 +33,7 @@ ENC_FILE_PATH = os.path.join(CRED_PATH, CRED_FILE_NAME_ENC) KEY_FILE = os.path.join(CRED_PATH, ".tmp_enc_key") -expected_completion_time_seconds = 240 +expected_completion_time_seconds = 400 # Check for credential path if not os.path.exists(CRED_PATH): diff --git a/src/api/rest/server/mcir/image.go b/src/api/rest/server/mcir/image.go index 00aa1d62f..422a5b369 100644 --- a/src/api/rest/server/mcir/image.go +++ b/src/api/rest/server/mcir/image.go @@ -78,7 +78,7 @@ func RestPostImage(c echo.Context) error { if err := c.Bind(u); err != nil { return common.EndRequestWithLog(c, reqID, err, nil) } - content, err := mcir.RegisterImageWithId(nsId, u, update) + content, err := mcir.RegisterImageWithId(nsId, u, update, false) return common.EndRequestWithLog(c, reqID, err, content) } else { err := fmt.Errorf("You must specify: action=registerWithInfo or action=registerWithId") @@ -116,7 +116,7 @@ func RestPutImage(c echo.Context) error { return common.EndRequestWithLog(c, reqID, err, nil) } - content, err := mcir.UpdateImage(nsId, resourceId, *u) + content, err := mcir.UpdateImage(nsId, resourceId, *u, false) return common.EndRequestWithLog(c, reqID, err, content) } diff --git a/src/core/mcir/common.go b/src/core/mcir/common.go index 02c3d85ac..44aea6390 100644 --- a/src/core/mcir/common.go +++ b/src/core/mcir/common.go @@ -1404,7 +1404,7 @@ func LoadCommonResource() (common.IdList, error) { regiesteredStatus := "" // WaitGroups for goroutine - var waitSpecImg sync.WaitGroup + // var waitSpecImg sync.WaitGroup var wait sync.WaitGroup // Check common namespace. Create one if not. @@ -1571,239 +1571,239 @@ func LoadCommonResource() (common.IdList, error) { } rowsImg = newRowsImg - waitSpecImg.Add(1) - go func(rowsSpec [][]string) { - defer waitSpecImg.Done() - //lenSpecs := len(rowsSpec[1:]) - for i, row := range rowsSpec[1:] { - // wait.Add(1) - // go func(i int, row []string, lenSpecs int) { - // defer wait.Done() - // common.RandomSleep(0, lenSpecs/20) - - specReqTmp := TbSpecReq{} - // 0 providerName - // 1 regionName - // 2 cspSpecName - // 3 CostPerHour - // 4 evaluationScore01 - // 5 evaluationScore02 - // 6 evaluationScore03 - // 7 evaluationScore04 - // 8 evaluationScore05 - // 9 evaluationScore06 - // 10 evaluationScore07 - // 11 evaluationScore08 - // 12 evaluationScore09 - // 13 evaluationScore10 - // 14 rootDiskType - // 15 rootDiskSize - // 17 acceleratorModel - // 18 acceleratorCount - // 19 acceleratorMemoryGB - // 20 acceleratorDetails - // 21 infraType + // waitSpecImg.Add(1) + //go func(rowsSpec [][]string) { + // defer waitSpecImg.Done() + //lenSpecs := len(rowsSpec[1:]) + for i, row := range rowsSpec[1:] { + // wait.Add(1) + // go func(i int, row []string, lenSpecs int) { + // defer wait.Done() + // common.RandomSleep(0, lenSpecs/20) + + specReqTmp := TbSpecReq{} + // 0 providerName + // 1 regionName + // 2 cspSpecName + // 3 CostPerHour + // 4 evaluationScore01 + // 5 evaluationScore02 + // 6 evaluationScore03 + // 7 evaluationScore04 + // 8 evaluationScore05 + // 9 evaluationScore06 + // 10 evaluationScore07 + // 11 evaluationScore08 + // 12 evaluationScore09 + // 13 evaluationScore10 + // 14 rootDiskType + // 15 rootDiskSize + // 17 acceleratorModel + // 18 acceleratorCount + // 19 acceleratorMemoryGB + // 20 acceleratorDetails + // 21 infraType + + providerName := strings.ToLower(row[0]) + regionName := strings.ToLower(row[1]) + specReqTmp.CspSpecName = row[2] + rootDiskType := row[14] + rootDiskSize := row[15] + acceleratorType := row[16] + acceleratorModel := row[17] + acceleratorCount := 0 + if s, err := strconv.Atoi(row[18]); err == nil { + acceleratorCount = s + } + acceleratorMemoryGB := 0.0 + if s, err := strconv.ParseFloat(row[19], 32); err == nil { + acceleratorMemoryGB = s + } + description := row[20] + infraType := strings.ToLower(row[21]) + + specReqTmp.Name = GetProviderRegionZoneResourceKey(providerName, regionName, "", specReqTmp.CspSpecName) + + //get connetion for lookup (if regionName is "all", use providerName only) + validRepresentativeConnectionMapKey := providerName + "-" + regionName + connectionForLookup, ok := validRepresentativeConnectionMap.Load(validRepresentativeConnectionMapKey) + if ok { + specReqTmp.ConnectionName = connectionForLookup.(common.ConnConfig).ConfigName + + _, ignoreCase := ignoreConnectionMap.Load(specReqTmp.ConnectionName) + if !ignoreCase { + // Give a name for spec object by combining ConnectionName and CspSpecName + // To avoid naming-rule violation, modify the string + + // specReqTmp.Name = specReqTmp.ConnectionName + "-" + specReqTmp.CspSpecName + // specReqTmp.Name = ToNamingRuleCompatible(specReqTmp.Name) + specInfoId := specReqTmp.Name + + specReqTmp.Description = "Common Spec Resource" + + regiesteredStatus = "" + + var errRegisterSpec error + + log.Trace().Msgf("[%d] register Common Spec: %s", i, specReqTmp.Name) + + // Register Spec object + searchKey := GetProviderRegionZoneResourceKey(providerName, regionName, "", specReqTmp.CspSpecName) + _, ok := specMap.Load(searchKey) + if ok { + // spiderSpec := value.(SpiderSpecInfo) + // //log.Info().Msgf("Found spec in the map: %s", spiderSpec.Name) + // tumblebugSpec, errConvert := ConvertSpiderSpecToTumblebugSpec(spiderSpec) + // if errConvert != nil { + // log.Error().Err(errConvert).Msg("Cannot ConvertSpiderSpecToTumblebugSpec") + // } + + // tumblebugSpec.Name = specInfoId + // tumblebugSpec.ConnectionName = specReqTmp.ConnectionName + // // _, errRegisterSpec = RegisterSpecWithInfo(common.SystemCommonNs, &tumblebugSpec, true) + // // if errRegisterSpec != nil { + // // log.Info().Err(errRegisterSpec).Msg("RegisterSpec WithInfo failed") + // // } + + } else { + errRegisterSpec = fmt.Errorf("Not Found spec from the fetched spec list: %s", searchKey) + log.Trace().Msgf(errRegisterSpec.Error()) + // _, errRegisterSpec = RegisterSpecWithCspSpecName(common.SystemCommonNs, &specReqTmp, true) + // if errRegisterSpec != nil { + // log.Error().Err(errRegisterSpec).Msg("RegisterSpec WithCspSpecName failed") + // } + } + if errRegisterSpec != nil { + regiesteredStatus += " [Failed] " + errRegisterSpec.Error() + } else { + // Update registered Spec object with givn info from asset file + // Update registered Spec object with Cost info + costPerHour, err2 := strconv.ParseFloat(strings.ReplaceAll(row[3], " ", ""), 32) + if err2 != nil { + log.Error().Msgf("Not valid CostPerHour value in the asset: %s", specInfoId) + costPerHour = 99999999.9 + } + evaluationScore01, err2 := strconv.ParseFloat(strings.ReplaceAll(row[4], " ", ""), 32) + if err2 != nil { + log.Error().Msgf("Not valid evaluationScore01 value in the asset: %s", specInfoId) + evaluationScore01 = -99.9 + } + expandedInfraType := expandInfraType(infraType) + specUpdateRequest := + TbSpecInfo{ + ProviderName: providerName, + RegionName: regionName, + CostPerHour: float32(costPerHour), + RootDiskType: rootDiskType, + RootDiskSize: rootDiskSize, + AcceleratorType: acceleratorType, + AcceleratorModel: acceleratorModel, + AcceleratorCount: uint8(acceleratorCount), + AcceleratorMemoryGB: float32(acceleratorMemoryGB), + Description: description, + EvaluationScore01: float32(evaluationScore01), + SystemLabel: "from-assets", + InfraType: expandedInfraType, + } + + _, err3 := UpdateSpec(common.SystemCommonNs, specInfoId, specUpdateRequest) + if err3 != nil { + log.Error().Err(err3).Msg("UpdateSpec failed") + regiesteredStatus += " [Failed] " + err3.Error() + } + //fmt.Printf("[%d] Registered Common Spec\n", i) + //common.PrintJsonPretty(updatedSpecInfo) + } + + regiesteredIds.AddItem(common.StrSpec + ": " + specInfoId + regiesteredStatus) + // }(i, row, lenSpecs) + } + } + } + // wait.Wait() + // }(rowsSpec) + + // // waitSpecImg.Add(1) + // go func(rowsImg [][]string) { + // // defer waitSpecImg.Done() + lenImages := len(rowsImg[1:]) + for i, row := range rowsImg[1:] { + wait.Add(1) + // fmt.Printf("[%d] i, row := range rowsImg[1:] %s\n", i, row) + // goroutine + go func(i int, row []string, lenImages int) { + defer wait.Done() + + imageReqTmp := TbImageReq{} + // row0: ProviderName + // row1: regionName + // row2: cspImageId + // row3: OsType + // row4: description + // row5: supportedInstance + // row6: infraType providerName := strings.ToLower(row[0]) regionName := strings.ToLower(row[1]) - specReqTmp.CspSpecName = row[2] - rootDiskType := row[14] - rootDiskSize := row[15] - acceleratorType := row[16] - acceleratorModel := row[17] - acceleratorCount := 0 - if s, err := strconv.Atoi(row[18]); err == nil { - acceleratorCount = s - } - acceleratorMemoryGB := 0.0 - if s, err := strconv.ParseFloat(row[19], 32); err == nil { - acceleratorMemoryGB = s - } - description := row[20] - infraType := strings.ToLower(row[21]) + imageReqTmp.CspImageId = row[2] + osType := strings.ReplaceAll(row[3], " ", "") + description := row[4] + infraType := strings.ToLower(row[6]) - specReqTmp.Name = GetProviderRegionZoneResourceKey(providerName, regionName, "", specReqTmp.CspSpecName) + // Give a name for spec object by combining ConnectionName and OsType + imageReqTmp.Name = GetProviderRegionZoneResourceKey(providerName, regionName, "", osType) //get connetion for lookup (if regionName is "all", use providerName only) validRepresentativeConnectionMapKey := providerName + "-" + regionName connectionForLookup, ok := validRepresentativeConnectionMap.Load(validRepresentativeConnectionMapKey) if ok { - specReqTmp.ConnectionName = connectionForLookup.(common.ConnConfig).ConfigName + imageReqTmp.ConnectionName = connectionForLookup.(common.ConnConfig).ConfigName - _, ignoreCase := ignoreConnectionMap.Load(specReqTmp.ConnectionName) + _, ignoreCase := ignoreConnectionMap.Load(imageReqTmp.ConnectionName) if !ignoreCase { - // Give a name for spec object by combining ConnectionName and CspSpecName - // To avoid naming-rule violation, modify the string - - // specReqTmp.Name = specReqTmp.ConnectionName + "-" + specReqTmp.CspSpecName - // specReqTmp.Name = ToNamingRuleCompatible(specReqTmp.Name) - specInfoId := specReqTmp.Name + // RandomSleep for safe parallel executions + common.RandomSleep(0, lenImages/8) - specReqTmp.Description = "Common Spec Resource" - - regiesteredStatus = "" - - var errRegisterSpec error + // To avoid naming-rule violation, modify the string + // imageReqTmp.Name = imageReqTmp.ConnectionName + "-" + osType + // imageReqTmp.Name = ToNamingRuleCompatible(imageReqTmp.Name) + imageInfoId := imageReqTmp.Name + imageReqTmp.Description = "Common Image Resource" - log.Trace().Msgf("[%d] register Common Spec: %s", i, specReqTmp.Name) + log.Trace().Msgf("[%d] register Common Image: %s", i, imageReqTmp.Name) // Register Spec object - searchKey := GetProviderRegionZoneResourceKey(providerName, regionName, "", specReqTmp.CspSpecName) - _, ok := specMap.Load(searchKey) - if ok { - // spiderSpec := value.(SpiderSpecInfo) - // //log.Info().Msgf("Found spec in the map: %s", spiderSpec.Name) - // tumblebugSpec, errConvert := ConvertSpiderSpecToTumblebugSpec(spiderSpec) - // if errConvert != nil { - // log.Error().Err(errConvert).Msg("Cannot ConvertSpiderSpecToTumblebugSpec") - // } - - // tumblebugSpec.Name = specInfoId - // tumblebugSpec.ConnectionName = specReqTmp.ConnectionName - // // _, errRegisterSpec = RegisterSpecWithInfo(common.SystemCommonNs, &tumblebugSpec, true) - // // if errRegisterSpec != nil { - // // log.Info().Err(errRegisterSpec).Msg("RegisterSpec WithInfo failed") - // // } - - } else { - errRegisterSpec = fmt.Errorf("Not Found spec from the fetched spec list: %s", searchKey) - log.Trace().Msgf(errRegisterSpec.Error()) - // _, errRegisterSpec = RegisterSpecWithCspSpecName(common.SystemCommonNs, &specReqTmp, true) - // if errRegisterSpec != nil { - // log.Error().Err(errRegisterSpec).Msg("RegisterSpec WithCspSpecName failed") - // } - } + regiesteredStatus = "" - if errRegisterSpec != nil { - regiesteredStatus += " [Failed] " + errRegisterSpec.Error() + _, err1 := RegisterImageWithId(common.SystemCommonNs, &imageReqTmp, true, true) + if err1 != nil { + log.Info().Msgf("Provider: %s, Region: %s, CspImageId: %s Error: %s", providerName, regionName, imageReqTmp.CspImageId, err1.Error()) + regiesteredStatus += " [Failed] " + err1.Error() } else { - // Update registered Spec object with givn info from asset file - // Update registered Spec object with Cost info - costPerHour, err2 := strconv.ParseFloat(strings.ReplaceAll(row[3], " ", ""), 32) - if err2 != nil { - log.Error().Msgf("Not valid CostPerHour value in the asset: %s", specInfoId) - costPerHour = 99999999.9 + // Update registered image object with OsType info + expandedInfraType := expandInfraType(infraType) + imageUpdateRequest := TbImageInfo{ + GuestOS: osType, + Description: description, + InfraType: expandedInfraType, } - evaluationScore01, err2 := strconv.ParseFloat(strings.ReplaceAll(row[4], " ", ""), 32) + _, err2 := UpdateImage(common.SystemCommonNs, imageInfoId, imageUpdateRequest, true) if err2 != nil { - log.Error().Msgf("Not valid evaluationScore01 value in the asset: %s", specInfoId) - evaluationScore01 = -99.9 - } - expandedInfraType := expandInfraType(infraType) - specUpdateRequest := - TbSpecInfo{ - ProviderName: providerName, - RegionName: regionName, - CostPerHour: float32(costPerHour), - RootDiskType: rootDiskType, - RootDiskSize: rootDiskSize, - AcceleratorType: acceleratorType, - AcceleratorModel: acceleratorModel, - AcceleratorCount: uint8(acceleratorCount), - AcceleratorMemoryGB: float32(acceleratorMemoryGB), - Description: description, - EvaluationScore01: float32(evaluationScore01), - SystemLabel: "from-assets", - InfraType: expandedInfraType, - } - - _, err3 := UpdateSpec(common.SystemCommonNs, specInfoId, specUpdateRequest) - if err3 != nil { - log.Error().Err(err3).Msg("UpdateSpec failed") - regiesteredStatus += " [Failed] " + err3.Error() + log.Error().Err(err2).Msg("UpdateImage failed") + regiesteredStatus += " [Failed] " + err2.Error() } - //fmt.Printf("[%d] Registered Common Spec\n", i) - //common.PrintJsonPretty(updatedSpecInfo) } - regiesteredIds.AddItem(common.StrSpec + ": " + specInfoId + regiesteredStatus) - // }(i, row, lenSpecs) + //regiesteredStatus = strings.Replace(regiesteredStatus, "\\", "", -1) + regiesteredIds.AddItem(common.StrImage + ": " + imageInfoId + regiesteredStatus) } } - } - wait.Wait() - }(rowsSpec) - - waitSpecImg.Add(1) - go func(rowsImg [][]string) { - defer waitSpecImg.Done() - lenImages := len(rowsImg[1:]) - for i, row := range rowsImg[1:] { - wait.Add(1) - // fmt.Printf("[%d] i, row := range rowsImg[1:] %s\n", i, row) - // goroutine - go func(i int, row []string, lenImages int) { - defer wait.Done() - - imageReqTmp := TbImageReq{} - // row0: ProviderName - // row1: regionName - // row2: cspImageId - // row3: OsType - // row4: description - // row5: supportedInstance - // row6: infraType - providerName := strings.ToLower(row[0]) - regionName := strings.ToLower(row[1]) - imageReqTmp.CspImageId = row[2] - osType := strings.ReplaceAll(row[3], " ", "") - description := row[4] - infraType := strings.ToLower(row[6]) - - // Give a name for spec object by combining ConnectionName and OsType - imageReqTmp.Name = GetProviderRegionZoneResourceKey(providerName, regionName, "", osType) - - //get connetion for lookup (if regionName is "all", use providerName only) - validRepresentativeConnectionMapKey := providerName + "-" + regionName - connectionForLookup, ok := validRepresentativeConnectionMap.Load(validRepresentativeConnectionMapKey) - if ok { - imageReqTmp.ConnectionName = connectionForLookup.(common.ConnConfig).ConfigName - - _, ignoreCase := ignoreConnectionMap.Load(imageReqTmp.ConnectionName) - if !ignoreCase { - // RandomSleep for safe parallel executions - common.RandomSleep(0, lenImages/8) - - // To avoid naming-rule violation, modify the string - // imageReqTmp.Name = imageReqTmp.ConnectionName + "-" + osType - // imageReqTmp.Name = ToNamingRuleCompatible(imageReqTmp.Name) - imageInfoId := imageReqTmp.Name - imageReqTmp.Description = "Common Image Resource" - - log.Trace().Msgf("[%d] register Common Image: %s", i, imageReqTmp.Name) - - // Register Spec object - regiesteredStatus = "" - - _, err1 := RegisterImageWithId(common.SystemCommonNs, &imageReqTmp, true) - if err1 != nil { - log.Info().Msgf("Provider: %s, Region: %s, CspImageId: %s Error: %s", providerName, regionName, imageReqTmp.CspImageId, err1.Error()) - regiesteredStatus += " [Failed] " + err1.Error() - } else { - // Update registered image object with OsType info - expandedInfraType := expandInfraType(infraType) - imageUpdateRequest := TbImageInfo{ - GuestOS: osType, - Description: description, - InfraType: expandedInfraType, - } - _, err2 := UpdateImage(common.SystemCommonNs, imageInfoId, imageUpdateRequest) - if err2 != nil { - log.Error().Err(err2).Msg("UpdateImage failed") - regiesteredStatus += " [Failed] " + err2.Error() - } - } - - //regiesteredStatus = strings.Replace(regiesteredStatus, "\\", "", -1) - regiesteredIds.AddItem(common.StrImage + ": " + imageInfoId + regiesteredStatus) - } - } - }(i, row, lenImages) - } - wait.Wait() - }(rowsImg) + }(i, row, lenImages) + } + wait.Wait() + // }(rowsImg) - waitSpecImg.Wait() + // waitSpecImg.Wait() // sort.Strings(regiesteredIds.IdList) log.Info().Msgf("Registered Common Resources %d", len(regiesteredIds.IdList)) @@ -2126,6 +2126,13 @@ func GetCspResourceId(nsId string, resourceType string, resourceId string) (stri } return specInfo.CspSpecName, nil } + if resourceType == common.StrImage { + imageInfo, err := GetImage(nsId, resourceId) + if err != nil { + return "", err + } + return imageInfo.CspImageId, nil + } key := common.GenResourceKey(nsId, resourceType, resourceId) if key == "/invalidKey" { @@ -2144,10 +2151,10 @@ func GetCspResourceId(nsId string, resourceType string, resourceId string) (stri } switch resourceType { - case common.StrImage: - content := mcirIds{} - json.Unmarshal([]byte(keyValue.Value), &content) - return content.CspImageId, nil + // case common.StrImage: + // content := mcirIds{} + // json.Unmarshal([]byte(keyValue.Value), &content) + // return content.CspImageId, nil case common.StrCustomImage: content := mcirIds{} json.Unmarshal([]byte(keyValue.Value), &content) diff --git a/src/core/mcir/image.go b/src/core/mcir/image.go index 7e52ba9bb..76dbbdd98 100644 --- a/src/core/mcir/image.go +++ b/src/core/mcir/image.go @@ -119,9 +119,8 @@ func ConvertSpiderImageToTumblebugImage(spiderImage SpiderImageInfo) (TbImageInf } // RegisterImageWithId accepts image creation request, creates and returns an TB image object -func RegisterImageWithId(nsId string, u *TbImageReq, update bool) (TbImageInfo, error) { +func RegisterImageWithId(nsId string, u *TbImageReq, update bool, RDBonly bool) (TbImageInfo, error) { - resourceType := common.StrImage content := TbImageInfo{} err := common.CheckString(nsId) @@ -130,38 +129,28 @@ func RegisterImageWithId(nsId string, u *TbImageReq, update bool) (TbImageInfo, return content, err } - // err = validate.Struct(u) - // if err != nil { - // if _, ok := err.(*validator.InvalidValidationError); ok { - // log.Err(err).Msg("") - // return content, err - // } - // return content, err - // } - - check, err := CheckResource(nsId, resourceType, u.Name) - - if !update { - if check { - err := fmt.Errorf("The image " + u.Name + " already exists.") + resourceType := common.StrImage + if !RDBonly { + check, err := CheckResource(nsId, resourceType, u.Name) + if !update { + if check { + err := fmt.Errorf("The image " + u.Name + " already exists.") + return content, err + } + } + if err != nil { + err := fmt.Errorf("Failed to check the existence of the image " + u.Name + ".") return content, err } } - if err != nil { - err := fmt.Errorf("Failed to check the existence of the image " + u.Name + ".") - return content, err - } - res, err := LookupImage(u.ConnectionName, u.CspImageId) if err != nil { log.Trace().Err(err).Msg("") - //err := fmt.Errorf("an error occurred while lookup image via CB-Spider") - return content, err } if res.IId.NameId == "" { - err := fmt.Errorf("CB-Spider returned empty IId.NameId with no error: %s", u.ConnectionName) + err := fmt.Errorf("CB-Spider returned empty IId.NameId without Error: %s", u.ConnectionName) log.Error().Err(err).Msgf("Cannot LookupImage %s %v", u.CspImageId, res) return content, err } @@ -178,19 +167,33 @@ func RegisterImageWithId(nsId string, u *TbImageReq, update bool) (TbImageInfo, content.Name = u.Name content.AssociatedObjectList = []string{} - //log.Info().Msg("PUT registerImage") - Key := common.GenResourceKey(nsId, resourceType, content.Id) - Val, _ := json.Marshal(content) - err = kvstore.Put(Key, string(Val)) - if err != nil { - log.Error().Err(err).Msg("") - return content, err + if !RDBonly { + Key := common.GenResourceKey(nsId, resourceType, content.Id) + Val, _ := json.Marshal(content) + err = kvstore.Put(Key, string(Val)) + if err != nil { + log.Error().Err(err).Msg("") + return content, err + } } // "INSERT INTO `image`(`namespace`, `id`, ...) VALUES ('nsId', 'content.Id', ...); - _, err = common.ORM.Insert(&content) + // Attempt to insert the new record + _, err = common.ORM.Insert(content) if err != nil { - log.Error().Err(err).Msg("") + if update { + // If insert fails and update is true, attempt to update the existing record + _, updateErr := common.ORM.Update(content, &TbSpecInfo{Namespace: content.Namespace, Id: content.Id}) + if updateErr != nil { + log.Error().Err(updateErr).Msg("Error updating spec after insert failure") + return content, updateErr + } else { + log.Trace().Msg("SQL: Update success after insert failure") + } + } else { + log.Error().Err(err).Msg("Error inserting spec and update flag is false") + return content, err + } } else { log.Trace().Msg("SQL: Insert success") } @@ -444,70 +447,116 @@ func SearchImage(nsId string, keywords ...string) ([]TbImageInfo, error) { // UpdateImage accepts to-be TB image objects, // updates and returns the updated TB image objects -func UpdateImage(nsId string, imageId string, fieldsToUpdate TbImageInfo) (TbImageInfo, error) { - resourceType := common.StrImage - temp := TbImageInfo{} - err := common.CheckString(nsId) - if err != nil { - log.Error().Err(err).Msg("") - return temp, err - } +func UpdateImage(nsId string, imageId string, fieldsToUpdate TbImageInfo, RDBonly bool) (TbImageInfo, error) { + if !RDBonly { - if len(fieldsToUpdate.Namespace) > 0 { - err := fmt.Errorf("You should not specify 'namespace' in the JSON request body.") - log.Error().Err(err).Msg("") - return temp, err - } + resourceType := common.StrImage + temp := TbImageInfo{} + err := common.CheckString(nsId) + if err != nil { + log.Error().Err(err).Msg("") + return temp, err + } - if len(fieldsToUpdate.Id) > 0 { - err := fmt.Errorf("You should not specify 'id' in the JSON request body.") - log.Error().Err(err).Msg("") - return temp, err - } + if len(fieldsToUpdate.Namespace) > 0 { + err := fmt.Errorf("You should not specify 'namespace' in the JSON request body.") + log.Error().Err(err).Msg("") + return temp, err + } + + if len(fieldsToUpdate.Id) > 0 { + err := fmt.Errorf("You should not specify 'id' in the JSON request body.") + log.Error().Err(err).Msg("") + return temp, err + } + + check, err := CheckResource(nsId, resourceType, imageId) + if err != nil { + log.Error().Err(err).Msg("") + return temp, err + } + + if !check { + err := fmt.Errorf("The image " + imageId + " does not exist.") + return temp, err + } + + tempInterface, err := GetResource(nsId, resourceType, imageId) + if err != nil { + err := fmt.Errorf("Failed to get the image " + imageId + ".") + return temp, err + } + asIsImage := TbImageInfo{} + err = common.CopySrcToDest(&tempInterface, &asIsImage) + if err != nil { + err := fmt.Errorf("Failed to CopySrcToDest() " + imageId + ".") + return temp, err + } + + // Update specified fields only + toBeImage := asIsImage + toBeImageJSON, _ := json.Marshal(fieldsToUpdate) + err = json.Unmarshal(toBeImageJSON, &toBeImage) + + Key := common.GenResourceKey(nsId, resourceType, toBeImage.Id) + Val, _ := json.Marshal(toBeImage) + err = kvstore.Put(Key, string(Val)) + if err != nil { + log.Error().Err(err).Msg("") + return temp, err + } - check, err := CheckResource(nsId, resourceType, imageId) + } + // "UPDATE `image` SET `id`='" + imageId + "', ... WHERE `namespace`='" + nsId + "' AND `id`='" + imageId + "';" + _, err := common.ORM.Update(&fieldsToUpdate, &TbSpecInfo{Namespace: nsId, Id: imageId}) if err != nil { log.Error().Err(err).Msg("") - return temp, err + return fieldsToUpdate, err + } else { + log.Trace().Msg("SQL: Update success") } - if !check { - err := fmt.Errorf("The image " + imageId + " does not exist.") - return temp, err + return fieldsToUpdate, nil +} + +// GetImage accepts namespace ID and imageKey(id,name,type,...), and returns the TB image object +func GetImage(nsId string, imageKey string) (TbImageInfo, error) { + if err := common.CheckString(nsId); err != nil { + log.Error().Err(err).Msg("Invalid namespace ID") + return TbImageInfo{}, err } - tempInterface, err := GetResource(nsId, resourceType, imageId) + log.Debug().Msg("[Get image] " + imageKey) + + // ex: tencent+ap-jakarta+ubuntu22.04 + image := TbImageInfo{Namespace: nsId, Id: imageKey} + has, err := common.ORM.Where("Namespace = ? AND Id = ?", nsId, imageKey).Get(&image) if err != nil { - err := fmt.Errorf("Failed to get the image " + imageId + ".") - return temp, err + log.Info().Err(err).Msgf("Failed to get image %s by ID", imageKey) } - asIsImage := TbImageInfo{} - err = common.CopySrcToDest(&tempInterface, &asIsImage) - if err != nil { - err := fmt.Errorf("Failed to CopySrcToDest() " + imageId + ".") - return temp, err + if has { + return image, nil } - // Update specified fields only - toBeImage := asIsImage - toBeImageJSON, _ := json.Marshal(fieldsToUpdate) - err = json.Unmarshal(toBeImageJSON, &toBeImage) - - Key := common.GenResourceKey(nsId, resourceType, toBeImage.Id) - Val, _ := json.Marshal(toBeImage) - err = kvstore.Put(Key, string(Val)) + // ex: img-487zeit5 + image = TbImageInfo{Namespace: nsId, CspImageId: imageKey} + has, err = common.ORM.Where("Namespace = ? AND CspImageId = ?", nsId, imageKey).Get(&image) if err != nil { - log.Error().Err(err).Msg("") - return temp, err + log.Info().Err(err).Msgf("Failed to get image %s by CspImageId", imageKey) + } + if has { + return image, nil } - // "UPDATE `image` SET `id`='" + imageId + "', ... WHERE `namespace`='" + nsId + "' AND `id`='" + imageId + "';" - _, err = common.ORM.Update(&toBeImage, &TbImageInfo{Namespace: nsId, Id: imageId}) + // ex: Ubuntu22.04 + image = TbImageInfo{Namespace: nsId, GuestOS: imageKey} + has, err = common.ORM.Where("Namespace = ? AND GuestOS = ?", nsId, imageKey).Get(&image) if err != nil { - log.Error().Err(err).Msg("") - } else { - log.Trace().Msg("SQL: Update success") + log.Info().Err(err).Msgf("Failed to get image %s by GuestOS type", imageKey) + } + if has { + return image, nil } - return toBeImage, nil + return TbImageInfo{}, fmt.Errorf("The imageKey %s not found by any of ID, CspImageId, GuestOS", imageKey) } diff --git a/src/core/mcis/provisioning.go b/src/core/mcis/provisioning.go index 8833ae9be..4ece74b01 100644 --- a/src/core/mcis/provisioning.go +++ b/src/core/mcis/provisioning.go @@ -1388,7 +1388,7 @@ func checkCommonResAvailable(req *TbVmDynamicReq) error { if strings.Contains(k.CommonImage, "+") { vmReq.ImageId = k.CommonImage } - _, err = mcir.GetResource(common.SystemCommonNs, common.StrImage, vmReq.ImageId) + _, err = mcir.GetImage(common.SystemCommonNs, vmReq.ImageId) if err != nil { err := fmt.Errorf("Failed to get Image " + k.CommonImage + " from " + vmReq.ConnectionName) log.Error().Err(err).Msg("") @@ -1441,7 +1441,7 @@ func getVmReqFromDynamicReq(reqID string, nsId string, req *TbVmDynamicReq) (*Tb if strings.Contains(k.CommonImage, "+") { vmReq.ImageId = k.CommonImage } - _, err = mcir.GetResource(common.SystemCommonNs, common.StrImage, vmReq.ImageId) + _, err = mcir.GetImage(common.SystemCommonNs, vmReq.ImageId) if err != nil { err := fmt.Errorf("Failed to get the Image " + vmReq.ImageId + " from " + vmReq.ConnectionName) log.Error().Err(err).Msg("") diff --git a/src/core/mcis/recommendation.go b/src/core/mcis/recommendation.go index cc47151b7..643f90ddb 100644 --- a/src/core/mcis/recommendation.go +++ b/src/core/mcis/recommendation.go @@ -160,6 +160,15 @@ func RecommendVm(nsId string, plan DeploymentPlan) ([]mcir.TbSpecInfo, error) { if len(filteredSpecs) == 0 { return []mcir.TbSpecInfo{}, nil } + // // sorting based on VCPU and MemoryGiB + // sort.Slice(filteredSpecs, func(i, j int) bool { + // // sort based on VCPU first + // if filteredSpecs[i].VCPU != filteredSpecs[j].VCPU { + // return float32(filteredSpecs[i].VCPU) < float32(filteredSpecs[j].VCPU) + // } + // // if VCPU is same, sort based on MemoryGiB + // return float32(filteredSpecs[i].MemoryGiB) < float32(filteredSpecs[j].MemoryGiB) + // }) // Prioritizing log.Debug().Msg("[Prioritizing specs]") @@ -280,18 +289,23 @@ func RecommendVmLatency(nsId string, specList *[]mcir.TbSpecInfo, param *[]Param //result[i].OrderInFilteredResult = uint16(i + 1) } - // if evaluations for distance are same, low cost will have priolity + // Sorting result based on multiple criteria: OrderInFilteredResult, CostPerHour, VCPU, MemoryGiB sort.Slice(result, func(i, j int) bool { - if result[i].OrderInFilteredResult < result[j].OrderInFilteredResult { - return true - } else if result[i].OrderInFilteredResult > result[j].OrderInFilteredResult { - return false - } else { + // 1st priority: OrderInFilteredResult + if result[i].OrderInFilteredResult != result[j].OrderInFilteredResult { + return result[i].OrderInFilteredResult < result[j].OrderInFilteredResult + } + // 2nd priority: CostPerHour + if result[i].CostPerHour != result[j].CostPerHour { return result[i].CostPerHour < result[j].CostPerHour } - //return result[i].OrderInFilteredResult < result[j].OrderInFilteredResult + // 3rd priority: VCPU + if result[i].VCPU != result[j].VCPU { + return float32(result[i].VCPU) < float32(result[j].VCPU) + } + // 4th priority: MemoryGiB + return float32(result[i].MemoryGiB) < float32(result[j].MemoryGiB) }) - // fmt.Printf("\n result : %v \n", result) // updatedSpec, err := mcir.UpdateSpec(nsId, *result) // content, err = mcir.SortSpecs(*specList, "memoryGiB", "descending") @@ -508,16 +522,22 @@ func RecommendVmLocation(nsId string, specList *[]mcir.TbSpecInfo, param *[]Para result := append([]mcir.TbSpecInfo{}, (*specList)...) - // if evaluations for distance are same, low cost will have priolity + // Sorting result based on multiple criteria: OrderInFilteredResult, CostPerHour, VCPU, MemoryGiB sort.Slice(result, func(i, j int) bool { - if result[i].OrderInFilteredResult < result[j].OrderInFilteredResult { - return true - } else if result[i].OrderInFilteredResult > result[j].OrderInFilteredResult { - return false - } else { + // 1st priority: OrderInFilteredResult + if result[i].OrderInFilteredResult != result[j].OrderInFilteredResult { + return result[i].OrderInFilteredResult < result[j].OrderInFilteredResult + } + // 2nd priority: CostPerHour + if result[i].CostPerHour != result[j].CostPerHour { return result[i].CostPerHour < result[j].CostPerHour } - //return result[i].OrderInFilteredResult < result[j].OrderInFilteredResult + // 3rd priority: VCPU + if result[i].VCPU != result[j].VCPU { + return float32(result[i].VCPU) < float32(result[j].VCPU) + } + // 4th priority: MemoryGiB + return float32(result[i].MemoryGiB) < float32(result[j].MemoryGiB) }) // fmt.Printf("\n result : %v \n", result) @@ -602,8 +622,6 @@ func RecommendVmCost(nsId string, specList *[]mcir.TbSpecInfo) ([]mcir.TbSpecInf result[i].EvaluationScore09 = float32((Max - result[i].CostPerHour) / (Max - Min + 0.0000001)) // Add small value to avoid NaN by division } - fmt.Printf("\n result : %v \n", result) - return result, nil } @@ -621,7 +639,6 @@ func RecommendVmPerformance(nsId string, specList *[]mcir.TbSpecInfo) ([]mcir.Tb result[i].OrderInFilteredResult = uint16(i + 1) result[i].EvaluationScore09 = float32((result[i].EvaluationScore01 - Min) / (Max - Min + 0.0000001)) // Add small value to avoid NaN by division } - fmt.Printf("\n result : %v \n", result) return result, nil } diff --git a/src/main.go b/src/main.go index cb6fecdf0..6eccb662d 100644 --- a/src/main.go +++ b/src/main.go @@ -317,7 +317,7 @@ func setConfig() { credViper.SetConfigType("yaml") err = credViper.ReadInConfig() if err != nil { - log.Info().Err(err).Msg("") + log.Info().Msg("Local credentials file not found. Continue.") } else { log.Info().Msg(credViper.ConfigFileUsed()) err = credViper.Unmarshal(&common.RuntimeCredential)