Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add custom groups support to upgrade run resource #1191

Merged
merged 3 commits into from
Jun 6, 2024
Merged

Add custom groups support to upgrade run resource #1191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b2715c4
Add custom groups support to upgrade run resource
ksamoray Apr 9, 2024
d235730
Use one function call to retrieve group list
ksamoray May 19, 2024
1996125
Host upgrade group cleanup when not in schema
ksamoray Jun 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 3 additions & 32 deletions nsxt/resource_nsxt_edge_transport_node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,23 +303,7 @@ func getEdgeNodeSettingsSchema() *schema.Schema {
Required: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"advanced_configuration": {
Type: schema.TypeList,
Optional: true,
Description: "Advanced configuration",
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"key": {
Type: schema.TypeString,
Required: true,
},
"value": {
Type: schema.TypeString,
Required: true,
},
},
},
},
"advanced_configuration": getKeyValuePairListSchema(),
"allow_ssh_root_login": {
Type: schema.TypeBool,
Default: false,
Expand Down Expand Up @@ -858,13 +842,7 @@ func getEdgeNodeSettingsFromSchema(s interface{}) (*model.EdgeNodeSettings, erro
settings := s.([]interface{})
for _, settingIf := range settings {
setting := settingIf.(map[string]interface{})
var advCfg []model.KeyValuePair
for _, aci := range setting["advanced_configuration"].([]interface{}) {
ac := aci.(map[string]interface{})
key := ac["key"].(string)
val := ac["value"].(string)
advCfg = append(advCfg, model.KeyValuePair{Key: &key, Value: &val})
}
advCfg := getKeyValuePairListFromSchema(setting["advanced_configuration"])
allowSSHRootLogin := setting["allow_ssh_root_login"].(bool)
dnsServers := interface2StringList(setting["dns_servers"].([]interface{}))
enableSSH := setting["enable_ssh"].(bool)
Expand Down Expand Up @@ -1255,14 +1233,7 @@ func resourceNsxtEdgeTransportNodeRead(d *schema.ResourceData, m interface{}) er

func setEdgeNodeSettingsInSchema(d *schema.ResourceData, nodeSettings *model.EdgeNodeSettings) error {
elem := getElemOrEmptyMapFromSchema(d, "node_settings")
var advCfg []map[string]interface{}
for _, kv := range nodeSettings.AdvancedConfiguration {
e := make(map[string]interface{})
e["key"] = kv.Key
e["value"] = kv.Value
advCfg = append(advCfg, e)
}
elem["advanced_configuration"] = advCfg
elem["advanced_configuration"] = setKeyValueListForSchema(nodeSettings.AdvancedConfiguration)
elem["allow_ssh_root_login"] = nodeSettings.AllowSshRootLogin
elem["dns_servers"] = nodeSettings.DnsServers
elem["enable_ssh"] = nodeSettings.EnableSsh
Expand Down
219 changes: 209 additions & 10 deletions nsxt/resource_nsxt_upgrade_run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,19 @@ import (
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/vmware/vsphere-automation-sdk-go/lib/vapi/std/errors"
"github.com/vmware/vsphere-automation-sdk-go/runtime/bindings"
"github.com/vmware/vsphere-automation-sdk-go/runtime/protocol/client"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx/fabric"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx/model"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx/upgrade"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp/nsx/upgrade/plan"
"golang.org/x/exp/slices"
)

const hostUpgradeUnitDefaultGroup = "Group 1 for ESXI"

// Order matters
var upgradeComponentList = []string{
edgeUpgradeGroup,
Expand Down Expand Up @@ -232,7 +238,8 @@ func getUpgradeGroupSchema(isHostGroup bool) *schema.Schema {
"id": {
Type: schema.TypeString,
Description: "ID of upgrade unit group",
Required: true,
Required: !isHostGroup,
Optional: isHostGroup,
},
"enabled": {
Type: schema.TypeBool,
Expand All @@ -255,6 +262,11 @@ func getUpgradeGroupSchema(isHostGroup bool) *schema.Schema {
}

if isHostGroup {
elemSchema["display_name"] = &schema.Schema{
Type: schema.TypeString,
Description: "Name of upgrade unit group",
Optional: true,
}
elemSchema["upgrade_mode"] = &schema.Schema{
Type: schema.TypeString,
Description: "Upgrade mode",
Expand All @@ -279,6 +291,15 @@ func getUpgradeGroupSchema(isHostGroup bool) *schema.Schema {
Optional: true,
Default: true,
}
elemSchema["hosts"] = &schema.Schema{
Type: schema.TypeList,
Description: "Hosts to be included in the upgrade group",
Optional: true,
Computed: true,
Elem: &schema.Schema{
Type: schema.TypeString,
},
}
}

return &schema.Schema{
Expand Down Expand Up @@ -340,7 +361,7 @@ func upgradeRunCreateOrUpdate(d *schema.ResourceData, m interface{}) error {
upgradeClientSet := newUpgradeClientSet(connector, d)

log.Printf("[INFO] Updating UpgradeUnitGroup and UpgradePlanSetting.")
err := prepareUpgrade(upgradeClientSet, d)
err := prepareUpgrade(upgradeClientSet, d, m)
if err != nil {
return handleCreateError("NsxtUpgradeRun", id, err)
}
Expand All @@ -358,7 +379,7 @@ func upgradeRunCreateOrUpdate(d *schema.ResourceData, m interface{}) error {
return resourceNsxtUpgradeRunRead(d, m)
}

func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData) error {
func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, m interface{}) error {
for i := range upgradeComponentList {
component := upgradeComponentList[i]
// Customize MP upgrade is not allowed
Expand Down Expand Up @@ -395,7 +416,7 @@ func prepareUpgrade(upgradeClientSet *upgradeClientSet, d *schema.ResourceData)
return err
}

err = updateUpgradeUnitGroups(upgradeClientSet, d, component)
err = updateUpgradeUnitGroups(upgradeClientSet, d, m, component)
if err != nil {
return err
}
Expand Down Expand Up @@ -490,7 +511,7 @@ func waitUpgradeForStatus(upgradeClientSet *upgradeClientSet, component *string,
return nil
}

func updateUpgradeUnitGroups(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, component string) error {
func updateUpgradeUnitGroups(upgradeClientSet *upgradeClientSet, d *schema.ResourceData, m interface{}, component string) error {
isBefore := false
getReorderAfterReq := func(id string) model.ReorderRequest {
return model.ReorderRequest{
Expand All @@ -499,13 +520,92 @@ func updateUpgradeUnitGroups(upgradeClientSet *upgradeClientSet, d *schema.Resou
}
}

groupList, err := upgradeClientSet.GroupClient.List(nil, nil, nil, nil, nil, nil, nil, nil)
if err != nil {
return err
}
groupIDMap := make(map[string]bool)
groupNameMap := make(map[string]bool)

preUpgradeGroupID := ""
for _, groupI := range d.Get(componentToGroupKey[component]).([]interface{}) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like the code doesn't delete custom groups that were created with previous apply (which might have failed) but are not present in current config, is that correct?
To solve this, d.GetChange("host_group") might be useful

group := groupI.(map[string]interface{})
groupID := group["id"].(string)
groupGet, err := upgradeClientSet.GroupClient.Get(groupID, nil)
if err != nil {
return err
var groupGet *model.UpgradeUnitGroup
var err error
isCreate := false
if groupID == "" {
groupName := group["display_name"].(string)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can the user change display_name for existing group?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No - this isn't doable as the display_name is used to identify a pre-existing custom group, and as the upgrade support doesn't maintain/support state.
If the upgrade implementation would support state, we could identify the group by id and update the group.

if groupName == "" {
return fmt.Errorf("couldn't find upgrade unit group without id or display_name")
}
groupNameMap[groupName] = true
// This is a custom group, try to find it by name
for i, group := range groupList.Results {
if *group.DisplayName == groupName {
if groupGet == nil {
groupID = *group.Id
groupGet = &groupList.Results[i]
} else {
return fmt.Errorf("upgrade group name %s is not unique", groupName)
}
}
}
if groupGet == nil {
// This is a new custom group, create an upgrade unit list
isCreate = true
var groupMembers []model.UpgradeUnit
if group["hosts"] != nil {
for _, h := range group["hosts"].([]interface{}) {
hostID := h.(string)
groupMembers = append(groupMembers, model.UpgradeUnit{Id: &hostID})
}
}
typeHost := "HOST"
groupGet = &model.UpgradeUnitGroup{DisplayName: &groupName, UpgradeUnits: groupMembers, Type_: &typeHost}
} else {
// This custom group might be updated, compare the upgrade unit lists
nsxUnits := getUnitIDsFromUnits(groupGet.UpgradeUnits)

// Find and remove upgrade units which have been removed from the list. This is done by assigning the
// upgrade unit to its default group
var schemaUnits []string
if group["hosts"] != nil {
schemaUnits = interface2StringList(group["hosts"].([]interface{}))
}
getHostDefaultUpgradeGroup, err := getHostDefaultUpgradeGroupGetter(m, groupList)
if err != nil {
return fmt.Errorf("failed to retrieve host upgrade groups, error is %v", err)
}
for _, nsxUnit := range nsxUnits {
if !slices.Contains(schemaUnits, nsxUnit) {
groupID, err := getHostDefaultUpgradeGroup(nsxUnit)
if isNotFoundError(err) {
return fmt.Errorf("couldn't find default group for host %s as default group was not found", nsxUnit)
} else if err != nil {
return handleUpdateError("Host Upgrade Group", nsxUnit, err)
}
err = addHostToGroup(m, groupID, nsxUnit, false)
if err != nil {
return handleUpdateError("Host Upgrade Group", nsxUnit, err)
}
}
}
// Replace the upgrade unit list
var groupMembers []model.UpgradeUnit
for _, h := range group["hosts"].([]interface{}) {
hostID := h.(string)
groupMembers = append(groupMembers, model.UpgradeUnit{Id: &hostID})
}
groupGet.UpgradeUnits = groupMembers
}
} else {
groupIDMap[groupID] = true
group, err := upgradeClientSet.GroupClient.Get(groupID, nil)
if err != nil {
return err
}
groupGet = &group
}

enabled := group["enabled"].(bool)
Expand Down Expand Up @@ -551,11 +651,14 @@ func updateUpgradeUnitGroups(upgradeClientSet *upgradeClientSet, d *schema.Resou
groupGet.ExtendedConfiguration = extendConfig
}

_, err = upgradeClientSet.GroupClient.Update(groupID, groupGet)
if isCreate {
_, err = upgradeClientSet.GroupClient.Create(*groupGet)
} else {
_, err = upgradeClientSet.GroupClient.Update(groupID, *groupGet)
}
if err != nil {
return err
}

if preUpgradeGroupID != "" {
err = upgradeClientSet.GroupClient.Reorder(groupID, getReorderAfterReq(preUpgradeGroupID))
if err != nil {
Expand All @@ -564,9 +667,37 @@ func updateUpgradeUnitGroups(upgradeClientSet *upgradeClientSet, d *schema.Resou
}
preUpgradeGroupID = groupID
}

// After group update is complete, check for empty custom host groups which aren't defined in the schema, and have no members - these can be deleted.
componentType := "HOST"
groupList, err = upgradeClientSet.GroupClient.List(&componentType, nil, nil, nil, nil, nil, nil, nil)
if err != nil {
return err
}
for _, group := range groupList.Results {
// Non-empty groups cannot be deleted anyway so skip
if *group.UpgradeUnitCount == 0 {
if !groupIDMap[*group.Id] && !groupNameMap[*group.DisplayName] && !isPredefinedGroup(m, group) {
err = upgradeClientSet.GroupClient.Delete(*group.Id)
if err != nil {
return err
}
}
}
}
return nil
}

func isPredefinedGroup(m interface{}, group model.UpgradeUnitGroup) bool {
if group.DisplayName != nil && *group.DisplayName == hostUpgradeUnitDefaultGroup && *group.Type_ == "HOST" {
return true
}
connector := getPolicyConnector(m)
client := fabric.NewComputeCollectionsClient(connector)
_, err := client.Get(*group.Id)
return !isNotFoundError(err)
}

func updateComponentUpgradePlanSetting(settingClient plan.SettingsClient, d *schema.ResourceData, component string) error {
settingI := d.Get(componentToSettingKey[component]).([]interface{})
if len(settingI) == 0 {
Expand Down Expand Up @@ -722,3 +853,71 @@ func resourceNsxtUpgradeRunUpdate(d *schema.ResourceData, m interface{}) error {
func resourceNsxtUpgradeRunDelete(d *schema.ResourceData, m interface{}) error {
return nil
}

func getHostDefaultUpgradeGroupGetter(m interface{}, groupList model.UpgradeUnitGroupListResult) (func(string) (string, error), error) {
connector := getPolicyConnector(m)
hostClient := nsx.NewTransportNodesClient(connector)

return func(hostID string) (string, error) {
host, err := hostClient.Get(hostID)
if err != nil {
return "", err
}
converter := bindings.NewTypeConverter()
base, errs := converter.ConvertToGolang(host.NodeDeploymentInfo, model.HostNodeBindingType())
if errs != nil {
return "", errs[0]
}
node := base.(model.HostNode)

if node.ComputeCollectionId != nil {
return *node.ComputeCollectionId, nil
}
// This host is not a part of a compute cluster:
// it should be assigned to the 'Group 1 for ESXI' group (this value is hardcoded in NSX)
if groupList.Results != nil {
for _, group := range groupList.Results {
if group.DisplayName != nil && *group.DisplayName == hostUpgradeUnitDefaultGroup && *group.Type_ == "HOST" {
return *group.Id, nil
}
}
}

return "", errors.NotFound{}
}, nil
}

func addHostToGroup(m interface{}, groupID, hostID string, isCreate bool) error {
connector := getPolicyConnector(m)
client := upgrade.NewUpgradeUnitGroupsClient(connector)

doUpdate := func() error {
group, err := client.Get(groupID, nil)
if err != nil {
return err
}

hostIDs := getUnitIDsFromUnits(group.UpgradeUnits)
if slices.Contains(hostIDs, hostID) {
// Host is already within the group
return nil
}
group.UpgradeUnits = append(group.UpgradeUnits, model.UpgradeUnit{Id: &hostID})
_, err = client.Update(groupID, group)
if err != nil {
return err
}
return nil
}
commonProviderConfig := getCommonProviderConfig(m)
return retryUponPreconditionFailed(doUpdate, commonProviderConfig.MaxRetries)
}

func getUnitIDsFromUnits(units []model.UpgradeUnit) []string {
var unitIDs []string

for _, unit := range units {
unitIDs = append(unitIDs, *unit.Id)
}
return unitIDs
}
Loading
Loading