Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor manager cluster resource #992

Merged
merged 1 commit into from
Oct 18, 2023
Merged

Refactor manager cluster resource #992

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 90 additions & 130 deletions nsxt/resource_nsxt_manager_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,71 +102,28 @@ func getClusterNodesFromSchema(d *schema.ResourceData) []NsxClusterNode {
return clusterNodes
}

func setNodesInSchema(d *schema.ResourceData, nodes []NsxClusterNode) error {
// Retrieve node credential from schema and set them in the nodeList element
// This is because the nodes are obtained using cluster client, and does not
// contain any information about credential
nodeIPToCredentialMap := getIPtoCredentialMap(d)

var nodeList []map[string]interface{}
for _, node := range nodes {
_, nodeInSchema := nodeIPToCredentialMap[node.IPAddress]
elem := make(map[string]interface{})
elem["id"] = node.ID
elem["ip_address"] = node.IPAddress
// If node is not in schema but in cluster node list,
// it means the node needs to be removed.
// In this case we don't need its credential because
// remove api does not require node credential
if nodeInSchema {
elem["username"] = nodeIPToCredentialMap[node.IPAddress][0]
elem["password"] = nodeIPToCredentialMap[node.IPAddress][1]
}
elem["fqdn"] = node.Fqdn
elem["status"] = node.Status
nodeList = append(nodeList, elem)
}
return d.Set("node", nodeList)
}

func getIPtoCredentialMap(d *schema.ResourceData) map[string][]string {
// returns a map, whose key is IP address of a node,
// value is a slice of [node_login_username, node_login_password]
nodes := d.Get("node").([]interface{})
var idToCredentialMap = map[string][]string{}
for _, node := range nodes {
data := node.(map[string]interface{})
ipAddress := data["ip_address"].(string)
userName := data["username"].(string)
password := data["password"].(string)
cred := []string{userName, password}
idToCredentialMap[ipAddress] = cred
}
return idToCredentialMap
}

func resourceNsxtManagerClusterCreate(d *schema.ResourceData, m interface{}) error {
// Call Joincluster function on nodes that are not in the cluster
nodes := getClusterNodesFromSchema(d)
if len(nodes) == 0 {
return fmt.Errorf("At least a manager appliance must be provided to form a cluster")
}
clusterID, certSha256Thumbprint, hostIP, err := getClusterInfoFromHostNode(d, m)
clusterID, certSha256Thumbprint, hostIPs, err := getClusterInfoFromHostNode(d, m)
if err != nil {
return handleCreateError("ManagerCluster", "", err)
}

for _, guestNode := range nodes {
err := joinNodeToCluster(clusterID, certSha256Thumbprint, guestNode, hostIP, d, m)
err := joinNodeToCluster(clusterID, certSha256Thumbprint, guestNode, hostIPs, d, m)
if err != nil {
return handleCreateError("ManagerCluster", hostIP, fmt.Errorf("failed to join node %s: %s", guestNode.ID, err))
return handleCreateError("ManagerCluster", clusterID, fmt.Errorf("failed to join node %s: %s", guestNode.ID, err))
}
}
d.SetId(clusterID)
return resourceNsxtManagerClusterRead(d, m)
}

func getClusterInfoFromHostNode(d *schema.ResourceData, m interface{}) (string, string, string, error) {
func getClusterInfoFromHostNode(d *schema.ResourceData, m interface{}) (string, string, []string, error) {
// function return values are:
// clusterID, certSha256Thumbprint, hostIP, error
connector := getPolicyConnector(m)
Expand All @@ -175,75 +132,57 @@ func getClusterInfoFromHostNode(d *schema.ResourceData, m interface{}) (string,
min := c.CommonConfig.MinRetryInterval
max := c.CommonConfig.MaxRetryInterval
maxRetries := c.CommonConfig.MaxRetries
hostIP := ""
hostIPs := []string{}
for i := 0; i < maxRetries; i++ {
clusterConfig, err := client.Get()
if err != nil {
return "", "", "", handleReadError(d, "Cluster Config", "", err)
return "", "", hostIPs, err
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if this fails after the first iteration we might return a non empty host IP list.
Is that expected?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be ok since error is always checked, and other return values are disregarded when error is present

}
if hostIP == "" {
hostIP, err = getHostIPFromClusterConfig(m, clusterConfig)
if len(hostIPs) == 0 {
hostIPs, err = resolveHostIPs(m)
if err != nil {
return "", "", "", err
return "", "", hostIPs, err
}

log.Printf("[DEBUG]: Host resolved to IP addresses %v", hostIPs)
}
clusterID := *clusterConfig.ClusterId
nodes := clusterConfig.Nodes
node := nodes[0]
apiListenAddr := node.ApiListenAddr
if apiListenAddr != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could it be better to first do this check, and only if it succeeds collect hostIPs? It seems while we wait for ApiListenAddr to populate we will keep repeating resolveHostIPs

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

resolveHostIPs only happens on first iteration, if len(hostIPs) == 0, so it shouldn't be repeated..

certSha256Thumbprint := *apiListenAddr.CertificateSha256Thumbprint
return clusterID, certSha256Thumbprint, hostIP, nil
return clusterID, certSha256Thumbprint, hostIPs, nil
}
interval := (rand.Intn(max-min) + min)
time.Sleep(time.Duration(interval) * time.Millisecond)
log.Printf("[DEBUG]: Waited %d ms before retrying getting API Listen Address, attempt %d", interval, i+1)
}
return "", "", "", fmt.Errorf("Failed to read ClusterConfig info from host node after %d attempts", maxRetries)
return "", "", hostIPs, fmt.Errorf("Failed to read ClusterConfig after %d attempts", maxRetries)
}

func getHostIPFromClusterConfig(client interface{}, clusterConfig nsxModel.ClusterConfig) (string, error) {
func resolveHostIPs(client interface{}) ([]string, error) {
c := client.(nsxtClients)
host := c.Host
host = strings.TrimPrefix(host, "https://")
// Check if host is ip address or fqdn, if it's ip address then we are all set,
// if it's fqdn, we loop through clusterconfig, find the node with same fqdn and
// return its ip address
// Check if host is ip address or fqdn, if it's ip address then we are all set
// Otherwise we resolve the host
ip := net.ParseIP(host)
isIPAddress := ip != nil
if isIPAddress {
return host, nil
if ip != nil {
return []string{host}, nil
}
nodes := clusterConfig.Nodes
for _, node := range nodes {
fqdn := *node.Fqdn
if fqdn == host {
if node.ApiListenAddr != nil {
// return ipv4 address if it has one, if it doesn't return ipv6 address
addr := node.ApiListenAddr
if *addr.IpAddress != "" {
return *addr.IpAddress, nil
}
return *addr.Ipv6Address, nil
}
v4, v6 := getHTTPSIPFromNodeEntity(node.Entities)
if v4 != "" {
return v4, nil
}
return v6, nil
}

ips, err := net.LookupIP(host)
if err != nil {
return nil, fmt.Errorf("Failed to resolve host ip from %s: %v", host, err)
}
return "", fmt.Errorf("Failed to get host ip from cluster config")
}

func getHTTPSIPFromNodeEntity(entities []nsxModel.NodeEntityInfo) (string, string) {
// returns the ipv4 and ipv6 address of a node's https port 443
for _, entity := range entities {
if *entity.Port == 443 {
return *entity.IpAddress, *entity.Ipv6Address
}
var result []string
for _, ip := range ips {
result = append(result, ip.String())
}
return "", ""

return result, nil
}

func getNewNsxtClient(node NsxClusterNode, d *schema.ResourceData, clients interface{}) (interface{}, error) {
Expand Down Expand Up @@ -282,7 +221,7 @@ func configureNewClient(newClient *nsxtClients, oldClient *nsxtClients, host str
return nil
}

func joinNodeToCluster(clusterID string, certSha256Thumbprint string, guestNode NsxClusterNode, masterNodeIP string, d *schema.ResourceData, m interface{}) error {
func joinNodeToCluster(clusterID string, certSha256Thumbprint string, guestNode NsxClusterNode, hostIPs []string, d *schema.ResourceData, m interface{}) error {
c, err := getNewNsxtClient(guestNode, d, m)
if err != nil {
return err
Expand All @@ -292,10 +231,14 @@ func joinNodeToCluster(clusterID string, certSha256Thumbprint string, guestNode
connector := getPolicyConnector(newNsxClients)
client := nsx.NewClusterClient(connector)
username, password := getHostCredential(m)
hostIP := getMatchingIPVersion(guestNode.IPAddress, hostIPs)
if hostIP == "" {
return fmt.Errorf("[ERROR] Failed to find matching IP version for the host in IP list %v", hostIPs)
}
joinClusterParams := nsxModel.JoinClusterParameters{
CertificateSha256Thumbprint: &certSha256Thumbprint,
ClusterId: &clusterID,
IpAddress: &masterNodeIP,
IpAddress: &hostIP,
Username: &username,
Password: &password,
}
Expand All @@ -314,6 +257,43 @@ func getHostCredential(m interface{}) (string, string) {
return username, password
}

func getMatchingIPVersion(ip string, hostIPs []string) string {
needIPv4 := (net.ParseIP(ip)).To4() != nil

for _, hostIP := range hostIPs {
isIPv4 := (net.ParseIP(hostIP)).To4() != nil
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are 100% sure hostIp is always a valid Ip address? otherwise we might first need to check ParseIP result.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We validate hostIP in resolveHostIPs, which is happening before this call

if needIPv4 == isIPv4 {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The code looks good but I find some difficulties to read it. It's not easy to understand that if needIPv4=False and isIPv4=False then we want to return an IPv6 address?

Do you agree or is it just me getting older?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Its a bit tricky to read, I agree, I've added a comment

// we return hostIP if either node ip is v4 and current host is v4,
// or node ip is v4 and current resolved host is v6
return hostIP
}
}

return ""
}

func isMatchingNode(node nsxModel.ClusterNodeInfo, address string) bool {
addr := net.ParseIP(address)
for _, entity := range node.Entities {
if entity.Port != nil {
if entity.IpAddress != nil {
nodeAddr := net.ParseIP(*entity.IpAddress)
if nodeAddr.Equal(addr) {
return true
}
}
if entity.Ipv6Address != nil {
nodeAddr := net.ParseIP(*entity.Ipv6Address)
if nodeAddr.Equal(addr) {
return true
}
}
}
}

return false
}

func resourceNsxtManagerClusterRead(d *schema.ResourceData, m interface{}) error {
id := d.Id()
connector := getPolicyConnector(m)
Expand All @@ -322,58 +302,38 @@ func resourceNsxtManagerClusterRead(d *schema.ResourceData, m interface{}) error
if err != nil {
return handleReadError(d, "ManagerCluster", id, err)
}
nodeInfo := clusterConfig.Nodes
var nodes []NsxClusterNode
hostIP, err := getHostIPFromClusterConfig(m, clusterConfig)
isIPv4 := (net.ParseIP(hostIP)).To4() != nil
if err != nil {
return handleReadError(d, "ManagerCluster", id, err)
}
for _, node := range nodeInfo {
ip := getIPFromNodeInfo(node, isIPv4)
fqdn := *node.Fqdn
status := *node.Status
if ip != hostIP {
id := *node.NodeUuid
clusterNode := NsxClusterNode{
ID: id,
IPAddress: ip,
Fqdn: fqdn,
Status: status,
nsxNodes := clusterConfig.Nodes
var resultNodes []map[string]interface{}
schemaNodes := getClusterNodesFromSchema(d)
// Complete schema nodes with computed fields
for _, schemaNode := range schemaNodes {
for _, nsxNode := range nsxNodes {
if isMatchingNode(nsxNode, schemaNode.IPAddress) {
resultNode := make(map[string]interface{})
resultNode["id"] = nsxNode.NodeUuid
resultNode["fqdn"] = nsxNode.Fqdn
resultNode["status"] = nsxNode.Status
resultNode["ip_address"] = schemaNode.IPAddress
resultNode["username"] = schemaNode.UserName
resultNode["password"] = schemaNode.Password

resultNodes = append(resultNodes, resultNode)
}
nodes = append(nodes, clusterNode)
}

}

d.Set("revision", clusterConfig.Revision)
setNodesInSchema(d, nodes)
d.Set("node", resultNodes)
return nil
}

func getIPFromNodeInfo(node nsxModel.ClusterNodeInfo, isIPv4 bool) string {
// After join node into cluster, apiListen address may take some time to become available
// if it's not available, we retrieve node ip from nodeEntityInfo
var ip *string
nodeEntities := node.Entities
for _, entity := range nodeEntities {
if *entity.Port == 443 {
if isIPv4 {
ip = entity.IpAddress
} else {
ip = entity.Ipv6Address
}
break
}
}
return *ip
}

func resourceNsxtManagerClusterUpdate(d *schema.ResourceData, m interface{}) error {
id := d.Id()
connector := getPolicyConnector(m)
client := nsx.NewClusterClient(connector)

clusterID, certSha256Thumbprint, hostIP, err := getClusterInfoFromHostNode(d, m)
clusterID, certSha256Thumbprint, hostIPs, err := getClusterInfoFromHostNode(d, m)
if err != nil {
return handleUpdateError("ManagerCluster", id, err)
}
Expand Down Expand Up @@ -406,7 +366,7 @@ func resourceNsxtManagerClusterUpdate(d *schema.ResourceData, m interface{}) err
UserName: userName,
Password: password,
}
err = joinNodeToCluster(clusterID, certSha256Thumbprint, nodeObj, hostIP, d, m)
err = joinNodeToCluster(clusterID, certSha256Thumbprint, nodeObj, hostIPs, d, m)
if err != nil {
return handleUpdateError("ManagerCluster", id, err)
}
Expand Down