Skip to content

Commit

Permalink
Refactor manager cluster resource
Browse files Browse the repository at this point in the history 
Main changes:
1. Resolve provider host address, if needed, rather than relying
on comparing host names in cluster config and in provider config
2. Rely on nodes IP version rather than host IP version, which
is unknown when not specified in provider config
3. Refactor Read function to update only Computed fields in state

Signed-off-by: Anna Khmelnitsky <[email protected]>
  • Loading branch information
@annakhm
annakhm committed Oct 5, 2023
1 parent 9d06ea9 commit 9fdbf6b
Showing 1 changed file with 79 additions and 127 deletions.
206 changes: 79 additions & 127 deletions nsxt/resource_nsxt_manager_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,71 +102,28 @@ func getClusterNodesFromSchema(d *schema.ResourceData) []NsxClusterNode {
return clusterNodes
}

func setNodesInSchema(d *schema.ResourceData, nodes []NsxClusterNode) error {
// Retrieve node credential from schema and set them in the nodeList element
// This is because the nodes are obtained using cluster client, and does not
// contain any information about credential
nodeIPToCredentialMap := getIPtoCredentialMap(d)

var nodeList []map[string]interface{}
for _, node := range nodes {
_, nodeInSchema := nodeIPToCredentialMap[node.IPAddress]
elem := make(map[string]interface{})
elem["id"] = node.ID
elem["ip_address"] = node.IPAddress
// If node is not in schema but in cluster node list,
// it means the node needs to be removed.
// In this case we don't need its credential because
// remove api does not require node credential
if nodeInSchema {
elem["username"] = nodeIPToCredentialMap[node.IPAddress][0]
elem["password"] = nodeIPToCredentialMap[node.IPAddress][1]
}
elem["fqdn"] = node.Fqdn
elem["status"] = node.Status
nodeList = append(nodeList, elem)
}
return d.Set("node", nodeList)
}

func getIPtoCredentialMap(d *schema.ResourceData) map[string][]string {
// returns a map, whose key is IP address of a node,
// value is a slice of [node_login_username, node_login_password]
nodes := d.Get("node").([]interface{})
var idToCredentialMap = map[string][]string{}
for _, node := range nodes {
data := node.(map[string]interface{})
ipAddress := data["ip_address"].(string)
userName := data["username"].(string)
password := data["password"].(string)
cred := []string{userName, password}
idToCredentialMap[ipAddress] = cred
}
return idToCredentialMap
}

func resourceNsxtManagerClusterCreate(d *schema.ResourceData, m interface{}) error {
// Call Joincluster function on nodes that are not in the cluster
nodes := getClusterNodesFromSchema(d)
if len(nodes) == 0 {
return fmt.Errorf("At least a manager appliance must be provided to form a cluster")
}
clusterID, certSha256Thumbprint, hostIP, err := getClusterInfoFromHostNode(d, m)
clusterID, certSha256Thumbprint, hostIPs, err := getClusterInfoFromHostNode(d, m)
if err != nil {
return handleCreateError("ManagerCluster", "", err)
}

for _, guestNode := range nodes {
err := joinNodeToCluster(clusterID, certSha256Thumbprint, guestNode, hostIP, d, m)
err := joinNodeToCluster(clusterID, certSha256Thumbprint, guestNode, hostIPs, d, m)
if err != nil {
return handleCreateError("ManagerCluster", hostIP, fmt.Errorf("failed to join node %s: %s", guestNode.ID, err))
return handleCreateError("ManagerCluster", clusterID, fmt.Errorf("failed to join node %s: %s", guestNode.ID, err))
}
}
d.SetId(clusterID)
return resourceNsxtManagerClusterRead(d, m)
}

func getClusterInfoFromHostNode(d *schema.ResourceData, m interface{}) (string, string, string, error) {
func getClusterInfoFromHostNode(d *schema.ResourceData, m interface{}) (string, string, []string, error) {
// function return values are:
// clusterID, certSha256Thumbprint, hostIP, error
connector := getPolicyConnector(m)
Expand All @@ -175,75 +132,58 @@ func getClusterInfoFromHostNode(d *schema.ResourceData, m interface{}) (string,
min := c.CommonConfig.MinRetryInterval
max := c.CommonConfig.MaxRetryInterval
maxRetries := c.CommonConfig.MaxRetries
hostIP := ""
hostIPs := []string{}
for i := 0; i < maxRetries; i++ {
clusterConfig, err := client.Get()
if err != nil {
return "", "", "", handleReadError(d, "Cluster Config", "", err)
return "", "", hostIPs, err
}
if hostIP == "" {
hostIP, err = getHostIPFromClusterConfig(m, clusterConfig)
if len(hostIPs) == 0 {
hostIPs, err = resolveHostIPs(m)
if err != nil {
return "", "", "", err
return "", "", hostIPs, err
}

log.Printf("[DEBUG]: Host resolved to IP addresses %v", hostIPs)
}
clusterID := *clusterConfig.ClusterId
nodes := clusterConfig.Nodes
node := nodes[0]
apiListenAddr := node.ApiListenAddr
if apiListenAddr != nil {
certSha256Thumbprint := *apiListenAddr.CertificateSha256Thumbprint
return clusterID, certSha256Thumbprint, hostIP, nil
return clusterID, certSha256Thumbprint, hostIPs, nil
}
interval := (rand.Intn(max-min) + min)
time.Sleep(time.Duration(interval) * time.Millisecond)
log.Printf("[DEBUG]: Waited %d ms before retrying getting API Listen Address, attempt %d", interval, i+1)
}
return "", "", "", fmt.Errorf("Failed to read ClusterConfig info from host node after %d attempts", maxRetries)
return "", "", hostIPs, fmt.Errorf("Failed to read ClusterConfig after %d attempts", maxRetries)
}

func getHostIPFromClusterConfig(client interface{}, clusterConfig nsxModel.ClusterConfig) (string, error) {
func resolveHostIPs(client interface{}) ([]string, error) {
c := client.(nsxtClients)
host := c.Host
host = strings.TrimPrefix(host, "https://")
// Check if host is ip address or fqdn, if it's ip address then we are all set,
// if it's fqdn, we loop through clusterconfig, find the node with same fqdn and
// return its ip address
ip := net.ParseIP(host)
isIPAddress := ip != nil
if isIPAddress {
return host, nil
if ip != nil {
return []string{host}, nil
}
nodes := clusterConfig.Nodes
for _, node := range nodes {
fqdn := *node.Fqdn
if fqdn == host {
if node.ApiListenAddr != nil {
// return ipv4 address if it has one, if it doesn't return ipv6 address
addr := node.ApiListenAddr
if *addr.IpAddress != "" {
return *addr.IpAddress, nil
}
return *addr.Ipv6Address, nil
}
v4, v6 := getHTTPSIPFromNodeEntity(node.Entities)
if v4 != "" {
return v4, nil
}
return v6, nil
}

ips, err := net.LookupIP(host)
if err != nil {
return nil, fmt.Errorf("Failed to resolve host ip from %s: %v", host, err)
}
return "", fmt.Errorf("Failed to get host ip from cluster config")
}

func getHTTPSIPFromNodeEntity(entities []nsxModel.NodeEntityInfo) (string, string) {
// returns the ipv4 and ipv6 address of a node's https port 443
for _, entity := range entities {
if *entity.Port == 443 {
return *entity.IpAddress, *entity.Ipv6Address
}
var result []string
for _, ip := range ips {
result = append(result, ip.String())
}
return "", ""

return result, nil
}

func getNewNsxtClient(node NsxClusterNode, d *schema.ResourceData, clients interface{}) (interface{}, error) {
Expand Down Expand Up @@ -282,7 +222,7 @@ func configureNewClient(newClient *nsxtClients, oldClient *nsxtClients, host str
return nil
}

func joinNodeToCluster(clusterID string, certSha256Thumbprint string, guestNode NsxClusterNode, masterNodeIP string, d *schema.ResourceData, m interface{}) error {
func joinNodeToCluster(clusterID string, certSha256Thumbprint string, guestNode NsxClusterNode, hostIPs []string, d *schema.ResourceData, m interface{}) error {
c, err := getNewNsxtClient(guestNode, d, m)
if err != nil {
return err
Expand All @@ -292,10 +232,14 @@ func joinNodeToCluster(clusterID string, certSha256Thumbprint string, guestNode
connector := getPolicyConnector(newNsxClients)
client := nsx.NewClusterClient(connector)
username, password := getHostCredential(m)
hostIP := getMatchingIPVersion(guestNode.IPAddress, hostIPs)
if hostIP == "" {
return fmt.Errorf("[ERROR] Failed to find matching IP version for the host in IP list %v", hostIPs)
}
joinClusterParams := nsxModel.JoinClusterParameters{
CertificateSha256Thumbprint: &certSha256Thumbprint,
ClusterId: &clusterID,
IpAddress: &masterNodeIP,
IpAddress: &hostIP,
Username: &username,
Password: &password,
}
Expand All @@ -314,6 +258,34 @@ func getHostCredential(m interface{}) (string, string) {
return username, password
}

func getMatchingIPVersion(ip string, hostIPs []string) string {
needIPv4 := (net.ParseIP(ip)).To4() != nil

for _, hostIP := range hostIPs {
isIPv4 := (net.ParseIP(hostIP)).To4() != nil
if needIPv4 == isIPv4 {
return hostIP
}
}

return ""
}

func isMatchingNode(node nsxModel.ClusterNodeInfo, address string) bool {
for _, entity := range node.Entities {
if entity.Port != nil {
if (entity.IpAddress != nil) && (*entity.IpAddress == address) {
return true
}
if (entity.Ipv6Address != nil) && (*entity.Ipv6Address == address) {
return true
}
}
}

return false
}

func resourceNsxtManagerClusterRead(d *schema.ResourceData, m interface{}) error {
id := d.Id()
connector := getPolicyConnector(m)
Expand All @@ -322,58 +294,38 @@ func resourceNsxtManagerClusterRead(d *schema.ResourceData, m interface{}) error
if err != nil {
return handleReadError(d, "ManagerCluster", id, err)
}
nodeInfo := clusterConfig.Nodes
var nodes []NsxClusterNode
hostIP, err := getHostIPFromClusterConfig(m, clusterConfig)
isIPv4 := (net.ParseIP(hostIP)).To4() != nil
if err != nil {
return handleReadError(d, "ManagerCluster", id, err)
}
for _, node := range nodeInfo {
ip := getIPFromNodeInfo(node, isIPv4)
fqdn := *node.Fqdn
status := *node.Status
if ip != hostIP {
id := *node.NodeUuid
clusterNode := NsxClusterNode{
ID: id,
IPAddress: ip,
Fqdn: fqdn,
Status: status,
nsxNodes := clusterConfig.Nodes
var resultNodes []map[string]interface{}
schemaNodes := getClusterNodesFromSchema(d)
// Complete schema nodes with computed fields
for _, schemaNode := range schemaNodes {
for _, nsxNode := range nsxNodes {
if isMatchingNode(nsxNode, schemaNode.IPAddress) {
resultNode := make(map[string]interface{})
resultNode["id"] = nsxNode.NodeUuid
resultNode["fqdn"] = nsxNode.Fqdn
resultNode["status"] = nsxNode.Status
resultNode["ip_address"] = schemaNode.IPAddress
resultNode["username"] = schemaNode.UserName
resultNode["password"] = schemaNode.Password

resultNodes = append(resultNodes, resultNode)
}
nodes = append(nodes, clusterNode)
}

}

d.Set("revision", clusterConfig.Revision)
setNodesInSchema(d, nodes)
d.Set("node", resultNodes)
return nil
}

func getIPFromNodeInfo(node nsxModel.ClusterNodeInfo, isIPv4 bool) string {
// After join node into cluster, apiListen address may take some time to become available
// if it's not available, we retrieve node ip from nodeEntityInfo
var ip *string
nodeEntities := node.Entities
for _, entity := range nodeEntities {
if *entity.Port == 443 {
if isIPv4 {
ip = entity.IpAddress
} else {
ip = entity.Ipv6Address
}
break
}
}
return *ip
}

func resourceNsxtManagerClusterUpdate(d *schema.ResourceData, m interface{}) error {
id := d.Id()
connector := getPolicyConnector(m)
client := nsx.NewClusterClient(connector)

clusterID, certSha256Thumbprint, hostIP, err := getClusterInfoFromHostNode(d, m)
clusterID, certSha256Thumbprint, hostIPs, err := getClusterInfoFromHostNode(d, m)
if err != nil {
return handleUpdateError("ManagerCluster", id, err)
}
Expand Down Expand Up @@ -406,7 +358,7 @@ func resourceNsxtManagerClusterUpdate(d *schema.ResourceData, m interface{}) err
UserName: userName,
Password: password,
}
err = joinNodeToCluster(clusterID, certSha256Thumbprint, nodeObj, hostIP, d, m)
err = joinNodeToCluster(clusterID, certSha256Thumbprint, nodeObj, hostIPs, d, m)
if err != nil {
return handleUpdateError("ManagerCluster", id, err)
}
Expand Down

0 comments on commit 9fdbf6b

Please sign in to comment.