Skip to content

Commit

Permalink
feat: drift detection for kuberenetes clusters and node pools
Browse files Browse the repository at this point in the history
  • Loading branch information
@goncalo-rodrigues
goncalo-rodrigues committed Aug 17, 2022
1 parent 9d941f8 commit d1df5b1
Show file tree
Hide file tree
Showing 15 changed files with 317 additions and 142 deletions.
28 changes: 21 additions & 7 deletions api/services/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func (s Service[Arg, OutT]) Create(ctx context.Context, in CreateRequest[Arg]) (
}

func (s Service[Arg, OutT]) create(ctx context.Context, in CreateRequest[Arg]) (out OutT, err error) {
done := false
key, err := util.ExtractApiKey(ctx)
if err != nil {
return
Expand Down Expand Up @@ -85,7 +86,7 @@ func (s Service[Arg, OutT]) create(ctx context.Context, in CreateRequest[Arg]) (
}

defer func() {
if err == nil {
if done && err == nil {
err = s.saveConfig(ctx, c, lock, configPrefix)
} else {
log.Println("[DEBUG] Something went wrong, not storing state")
Expand All @@ -98,12 +99,16 @@ func (s Service[Arg, OutT]) create(ctx context.Context, in CreateRequest[Arg]) (
return
}
defer func() {
if err != nil {
if err != nil || !done {
rollbackFn()
}
}()

return s.readFromConfig(ctx, c, &resourcespb.ReadVirtualNetworkRequest{ResourceId: resource.GetResourceId()}, configPrefix)
out, err = s.readFromConfig(ctx, c, &resourcespb.ReadVirtualNetworkRequest{ResourceId: resource.GetResourceId()}, configPrefix)

// this must always be the last statement, if it doesn't execute it means we panicked or returned and need to rollback
done = true
return
}

func (s Service[Arg, OutT]) getConfig(ctx context.Context, userId string, lock *db.ConfigLock, configPrefix string) (*resources.MultyConfig, error) {
Expand Down Expand Up @@ -198,6 +203,7 @@ func (s Service[Arg, OutT]) Update(ctx context.Context, in UpdateRequest[Arg]) (
}

func (s Service[Arg, OutT]) update(ctx context.Context, in UpdateRequest[Arg]) (out OutT, err error) {
done := false
key, err := util.ExtractApiKey(ctx)
if err != nil {
return
Expand Down Expand Up @@ -227,7 +233,7 @@ func (s Service[Arg, OutT]) update(ctx context.Context, in UpdateRequest[Arg]) (
}

defer func() {
if err == nil {
if err == nil && done {
err = s.saveConfig(ctx, c, lock, configPrefix)
} else {
log.Println("[DEBUG] Something went wrong, not storing state")
Expand All @@ -239,11 +245,15 @@ func (s Service[Arg, OutT]) update(ctx context.Context, in UpdateRequest[Arg]) (
return
}
defer func() {
if err != nil {
if err != nil || !done {
rollbackFn()
}
}()
return s.readFromConfig(ctx, c, in, configPrefix)
out, err = s.readFromConfig(ctx, c, in, configPrefix)

// this must always be the last statement, if it doesn't execute it means we panicked or returned and need to rollback
done = true
return
}

func (s Service[Arg, OutT]) Delete(ctx context.Context, in WithResourceId) (_ *commonpb.Empty, err error) {
Expand All @@ -252,6 +262,7 @@ func (s Service[Arg, OutT]) Delete(ctx context.Context, in WithResourceId) (_ *c
}

func (s Service[Arg, OutT]) delete(ctx context.Context, in WithResourceId) (out *commonpb.Empty, err error) {
done := false
key, err := util.ExtractApiKey(ctx)
if err != nil {
return
Expand All @@ -278,7 +289,7 @@ func (s Service[Arg, OutT]) delete(ctx context.Context, in WithResourceId) (out
}

defer func() {
if err == nil {
if err == nil && done {
err = s.saveConfig(ctx, c, lock, configPrefix)
} else {
log.Println("[DEBUG] Something went wrong, not storing state")
Expand All @@ -297,5 +308,8 @@ func (s Service[Arg, OutT]) delete(ctx context.Context, in WithResourceId) (out
}
return nil, err
}

// this must always be the last statement, if it doesn't execute it means we panicked or returned and need to rollback
done = true
return &commonpb.Empty{}, nil
}
10 changes: 10 additions & 0 deletions resources/common/vm_size.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,13 @@ func GetVmSize(s commonpb.VmSize_Enum, c commonpb.CloudProvider) (string, error)
return result, nil
}
}

func ParseVmSize(size string, c commonpb.CloudProvider) commonpb.VmSize_Enum {
for generalSize, perCloud := range VMSIZE {
if vmSize, ok := perCloud[c]; ok && vmSize == size {
return generalSize
}
}

return commonpb.VmSize_UNKNOWN_VM_SIZE
}
25 changes: 13 additions & 12 deletions resources/output/kubernetes_node_pool/aws_eks_node_group.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package kubernetes_node_pool
import "github.com/multycloud/multy/resources/common"

type ScalingConfig struct {
DesiredSize int `hcl:"desired_size"`
MaxSize int `hcl:"max_size"`
MinSize int `hcl:"min_size"`
DesiredSize int `hcl:"desired_size" json:"desired_size"`
MaxSize int `hcl:"max_size" json:"max_size"`
MinSize int `hcl:"min_size" json:"min_size"`
}

type UpdateConfig struct {
Expand All @@ -14,14 +14,15 @@ type UpdateConfig struct {

type AwsKubernetesNodeGroup struct {
*common.AwsResource `hcl:",squash" default:"name=aws_eks_node_group"`
ClusterName string `hcl:"cluster_name,expr"`
NodeGroupName string `hcl:"node_group_name"`
NodeRoleArn string `hcl:"node_role_arn,expr"`
SubnetIds []string `hcl:"subnet_ids,expr"`
ScalingConfig ScalingConfig `hcl:"scaling_config"`
UpdateConfig UpdateConfig `hcl:"update_config" hcle:"omitempty"`
Labels map[string]string `hcl:"labels" hcle:"omitempty"`
InstanceTypes []string `hcl:"instance_types"`
ClusterName string `hcl:"cluster_name,expr" json:"cluster_name"`
NodeGroupName string `hcl:"node_group_name" json:"node_group_name"`
NodeRoleArn string `hcl:"node_role_arn,expr" json:"node_role_arn"`
SubnetIds []string `hcl:"subnet_ids,expr" json:"subnet_ids"`
ScalingConfig []ScalingConfig `hcl:"scaling_config,blocks" json:"scaling_config"`
UpdateConfig []UpdateConfig `hcl:"update_config,blocks" hcle:"omitempty" json:"update_config"`
Labels map[string]string `hcl:"labels" hcle:"omitempty" json:"labels"`
InstanceTypes []string `hcl:"instance_types" json:"instance_types"`
DiskSize int `hcl:"disk_size" hcle:"omitempty" json:"disk_size"`

Arn string `json:"arn" hcle:"omitempty"`
Arn string `json:"arn" hcle:"omitempty" json:"arn"`
}
21 changes: 11 additions & 10 deletions resources/output/kubernetes_node_pool/azure_aks_node_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,15 @@ import "github.com/multycloud/multy/resources/common"

type AzureKubernetesNodePool struct {
*common.AzResource `hcl:",squash" default:"name=azurerm_kubernetes_cluster_node_pool"`
ClusterId string `hcl:"kubernetes_cluster_id,expr" hcle:"omitempty"`
Name string `hcl:"name,optional" hcle:"omitempty"`
NodeCount int `hcl:"node_count"`
MaxSize int `hcl:"max_count"`
MinSize int `hcl:"min_count"`
Labels map[string]string `hcl:"node_labels" hcle:"omitempty"`
EnableAutoScaling bool `hcl:"enable_auto_scaling"`
VmSize string `hcl:"vm_size"`
VirtualNetworkSubnetId string `hcl:"vnet_subnet_id,expr"`
Zones []string `hcl:"zones" hcle:"omitempty"`
ClusterId string `hcl:"kubernetes_cluster_id,expr" hcle:"omitempty" json:"kubernetes_cluster_id"`
Name string `hcl:"name,optional" hcle:"omitempty" json:"name"`
NodeCount int `hcl:"node_count" json:"node_count"`
MaxSize int `hcl:"max_count" json:"max_count"`
MinSize int `hcl:"min_count" json:"min_count"`
Labels map[string]string `hcl:"node_labels" hcle:"omitempty" json:"node_labels"`
EnableAutoScaling bool `hcl:"enable_auto_scaling" json:"enable_auto_scaling"`
VmSize string `hcl:"vm_size" json:"vm_size"`
VirtualNetworkSubnetId string `hcl:"vnet_subnet_id,expr" json:"vnet_subnet_id"`
Zones []string `hcl:"zones" hcle:"omitempty" json:"zones"`
OsDiskSizeGb int `hcl:"os_disk_size_gb" hcle:"omitempty" json:"os_disk_size_gb" `
}
38 changes: 19 additions & 19 deletions resources/output/kubernetes_node_pool/gcp_gke_node_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,33 @@ import "github.com/multycloud/multy/resources/common"

type GoogleContainerNodePool struct {
*common.GcpResource `hcl:",squash" default:"name=google_container_node_pool"`
Cluster string `hcl:"cluster,expr"` //expr
InitialNodeCount int `hcl:"initial_node_count"`
NodeLocations []string `hcl:"node_locations" hcle:"omitempty"`
Autoscaling GoogleContainerNodePoolAutoScaling `hcl:"autoscaling" json:"-"`
NodeConfig GoogleContainerNodeConfig `hcl:"node_config" json:"-"`
NetworkConfig GoogleContainerNetworkConfig `hcl:"network_config" hcle:"omitempty" json:"-"`
Cluster string `hcl:"cluster,expr" json:"cluster"` //expr
InitialNodeCount int `hcl:"initial_node_count" json:"initial_node_count"`
NodeLocations []string `hcl:"node_locations" hcle:"omitempty" json:"node_locations"`
Autoscaling []GoogleContainerNodePoolAutoScaling `hcl:"autoscaling,blocks" json:"autoscaling"`
NodeConfig []GoogleContainerNodeConfig `hcl:"node_config,blocks" json:"node_config"`
NetworkConfig []GoogleContainerNetworkConfig `hcl:"network_config,blocks" hcle:"omitempty" json:"network_config"`
}

type GoogleContainerNodePoolAutoScaling struct {
MinNodeCount int `hcl:"min_node_count"`
MaxNodeCount int `hcl:"max_node_count"`
MinNodeCount int `hcl:"min_node_count" json:"min_node_count"`
MaxNodeCount int `hcl:"max_node_count" json:"max_node_count"`
}

type GoogleContainerNodeConfig struct {
DiskSizeGb int `hcl:"disk_size_gb" hcle:"omitempty"`
DiskType string `hcl:"disk_type" hcle:"omitempty"`
ImageType string `hcl:"image_type" hcle:"omitempty"`
Labels map[string]string `hcl:"labels" hcle:"omitempty"`
MachineType string `hcl:"machine_type"`
Metadata map[string]string `hcl:"metadata" hcle:"omitempty"`
Tags []string `hcl:"tags" hcle:"omitempty"`
DiskSizeGb int `hcl:"disk_size_gb" hcle:"omitempty" json:"disk_size_gb"`
DiskType string `hcl:"disk_type" hcle:"omitempty" json:"disk_type"`
ImageType string `hcl:"image_type" hcle:"omitempty" json:"image_type"`
Labels map[string]string `hcl:"labels" hcle:"omitempty" json:"labels"`
MachineType string `hcl:"machine_type" json:"machine_type"`
Metadata map[string]string `hcl:"metadata" hcle:"omitempty" json:"metadata"`
Tags []string `hcl:"tags" hcle:"omitempty" json:"tags"`

ServiceAccount string `hcl:"service_account,expr"`
OAuthScopes []string `hcl:"oauth_scopes"`
ServiceAccount string `hcl:"service_account,expr" json:"service_account"`
OAuthScopes []string `hcl:"oauth_scopes" json:"o_auth_scopes"`
}

type GoogleContainerNetworkConfig struct {
CreatePodRange bool `hcl:"create_pod_range"`
PodIpv4CidrBlock string `hcl:"pod_ipv4_cidr_block"`
CreatePodRange bool `hcl:"create_pod_range" json:"create_pod_range"`
PodIpv4CidrBlock string `hcl:"pod_ipv4_cidr_block" json:"pod_ipv_4_cidr_block"`
}
12 changes: 6 additions & 6 deletions resources/output/kubernetes_service/aws_eks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ import (
)

type AwsEksCluster struct {
*common.AwsResource `hcl:",squash" default:"name=aws_eks_cluster"`
RoleArn string `hcl:"role_arn,expr"`
VpcConfig VpcConfig `hcl:"vpc_config"`
KubernetesNetworkConfig KubernetesNetworkConfig `hcl:"kubernetes_network_config"`
Name string `hcl:"name"`
*common.AwsResource `hcl:",squash" default:"name=aws_eks_cluster" json:"*_common_._aws_resource"`
RoleArn string `hcl:"role_arn,expr" json:"role_arn"`
VpcConfig []VpcConfig `hcl:"vpc_config,blocks" json:"vpc_config"`
KubernetesNetworkConfig []KubernetesNetworkConfig `hcl:"kubernetes_network_config,blocks" json:"kubernetes_network_config"`
Name string `hcl:"name" json:"name"`

// outputs
Endpoint string `json:"endpoint" hcle:"omitempty"`
Expand All @@ -23,7 +23,7 @@ type VpcConfig struct {
}

type KubernetesNetworkConfig struct {
ServiceIpv4Cidr string `hcl:"service_ipv4_cidr"`
ServiceIpv4Cidr string `hcl:"service_ipv4_cidr" json:"service_ipv_4_cidr"`
}

type CertificateAuthority struct {
Expand Down
21 changes: 11 additions & 10 deletions resources/output/kubernetes_service/azure_aks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,21 +14,22 @@ type AzureIdentity struct {

type AzureEksCluster struct {
*common.AzResource `hcl:",squash" default:"name=azurerm_kubernetes_cluster"`
DefaultNodePool *kubernetes_node_pool.AzureKubernetesNodePool `hcl:"default_node_pool"`
DnsPrefix string `hcl:"dns_prefix"`
Identity []AzureIdentity `hcl:"identity,blocks"`
NetworkProfile NetworkProfile `hcl:"network_profile"`
DefaultNodePool *kubernetes_node_pool.AzureKubernetesNodePool `hcl:"default_node_pool" json:"-"`
DnsPrefix string `hcl:"dns_prefix" json:"dns_prefix"`
Identity []AzureIdentity `hcl:"identity,blocks" json:"identity"`
NetworkProfile []NetworkProfile `hcl:"network_profile,blocks" json:"network_profile"`

// outputs
KubeConfigRaw string `json:"kube_config_raw" hcle:"omitempty"`
KubeConfig []AzureKubeConfig `json:"kube_config" hcle:"omitempty"`
KubeConfigRaw string `json:"kube_config_raw" hcle:"omitempty" json:"kube_config_raw"`
KubeConfig []AzureKubeConfig `json:"kube_config" hcle:"omitempty" json:"kube_config"`
DefaultNodePoolOut []*kubernetes_node_pool.AzureKubernetesNodePool `json:"default_node_pool"`
}

type NetworkProfile struct {
NetworkPlugin string `hcl:"network_plugin"`
DnsServiceIp string `hcl:"dns_service_ip"`
DockerBridgeCidr string `hcl:"docker_bridge_cidr"`
ServiceCidr string `hcl:"service_cidr"`
NetworkPlugin string `hcl:"network_plugin" json:"network_plugin"`
DnsServiceIp string `hcl:"dns_service_ip" json:"dns_service_ip"`
DockerBridgeCidr string `hcl:"docker_bridge_cidr" json:"docker_bridge_cidr"`
ServiceCidr string `hcl:"service_cidr" json:"service_cidr"`
}

type AzureKubeConfig struct {
Expand Down
18 changes: 9 additions & 9 deletions resources/output/kubernetes_service/gcp_gke.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,17 @@ import (

type GoogleContainerCluster struct {
*common.GcpResource `hcl:",squash" default:"name=google_container_cluster"`
RemoveDefaultNodePool bool `hcl:"remove_default_node_pool"`
InitialNodeCount int `hcl:"initial_node_count"`
Subnetwork string `hcl:"subnetwork,expr"`
Network string `hcl:"network,expr"`
IpAllocationPolicy GoogleContainerClusterIpAllocationPolicy `hcl:"ip_allocation_policy"`
Location string `hcl:"location"`
NodeConfig kubernetes_node_pool.GoogleContainerNodeConfig `hcl:"node_config"`
RemoveDefaultNodePool bool `hcl:"remove_default_node_pool" json:"remove_default_node_pool"`
InitialNodeCount int `hcl:"initial_node_count" json:"initial_node_count"`
Subnetwork string `hcl:"subnetwork,expr" json:"subnetwork"`
Network string `hcl:"network,expr" json:"network"`
IpAllocationPolicy []GoogleContainerClusterIpAllocationPolicy `hcl:"ip_allocation_policy,blocks" json:"ip_allocation_policy"`
Location string `hcl:"location" json:"location"`
NodeConfig []kubernetes_node_pool.GoogleContainerNodeConfig `hcl:"node_config,blocks" json:"node_config"`

// outputs
Endpoint string `json:"endpoint" hcle:"omitempty"`
MasterAuth []GoogleContainerClusterAuth `json:"master_auth" hcle:"omitempty"`
Endpoint string `json:"endpoint" hcle:"omitempty" json:"endpoint"`
MasterAuth []GoogleContainerClusterAuth `json:"master_auth" hcle:"omitempty" json:"master_auth"`
}

type GoogleContainerClusterAuth struct {
Expand Down
47 changes: 31 additions & 16 deletions resources/types/aws/kubernetes_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,29 +104,44 @@ func (r AwsKubernetesCluster) FromState(state *output.TfState) (*resourcespb.Kub
return result, nil
}

cluster, err := output.GetParsedById[kubernetes_service.AwsEksCluster](state, r.ResourceId)
if err != nil {
return nil, err
}
result.Endpoint = cluster.Endpoint
result.CaCertificate = cluster.CertificateAuthority[0].Data
kubeCgfRaw, err := createKubeConfig(r.Args.Name, result.CaCertificate, result.Endpoint, r.GetCloudSpecificLocation())
if err != nil {
return nil, err
}
result.KubeConfigRaw = kubeCgfRaw
statuses := map[string]commonpb.ResourceStatus_Status{}
result.AwsOutputs = &resourcespb.KubernetesClusterAwsOutputs{}

if cluster, exists, err := output.MaybeGetParsedById[kubernetes_service.AwsEksCluster](state, r.ResourceId); exists {
if err != nil {
return nil, err
}
result.Endpoint = cluster.Endpoint
result.CaCertificate = cluster.CertificateAuthority[0].Data
result.Name = cluster.Name
if len(cluster.KubernetesNetworkConfig) == 0 {
result.ServiceCidr = ""
} else {
result.ServiceCidr = cluster.KubernetesNetworkConfig[0].ServiceIpv4Cidr
}
kubeCgfRaw, err := createKubeConfig(r.Args.Name, result.CaCertificate, result.Endpoint, r.GetCloudSpecificLocation())
if err != nil {
return nil, err
}
result.KubeConfigRaw = kubeCgfRaw

result.AwsOutputs = &resourcespb.KubernetesClusterAwsOutputs{
EksClusterId: cluster.Arn,
result.AwsOutputs.EksClusterId = cluster.Arn
} else {
statuses["aws_eks_cluster"] = commonpb.ResourceStatus_NEEDS_CREATE
}

if stateResource, exists, err := output.MaybeGetParsedById[iam.AwsIamRole](state, r.ResourceId); exists {
if err != nil {
return nil, err
}
result.AwsOutputs.IamRoleArn = stateResource.Arn
} else {
statuses["aws_iam_role"] = commonpb.ResourceStatus_NEEDS_CREATE
}

if len(statuses) > 0 {
result.CommonParameters.ResourceStatus = &commonpb.ResourceStatus{Statuses: statuses}
}
return result, nil
}

Expand Down Expand Up @@ -180,11 +195,11 @@ func (r AwsKubernetesCluster) Translate(ctx resources.MultyContext) ([]output.Tf
&kubernetes_service.AwsEksCluster{
AwsResource: common.NewAwsResourceWithDeps(r.ResourceId, r.Args.Name, deps),
RoleArn: fmt.Sprintf("aws_iam_role.%s.arn", r.ResourceId),
VpcConfig: kubernetes_service.VpcConfig{SubnetIds: subnetIds, EndpointPrivateAccess: true},
VpcConfig: []kubernetes_service.VpcConfig{{SubnetIds: subnetIds, EndpointPrivateAccess: true}},
Name: r.Args.Name,
KubernetesNetworkConfig: kubernetes_service.KubernetesNetworkConfig{
KubernetesNetworkConfig: []kubernetes_service.KubernetesNetworkConfig{{
ServiceIpv4Cidr: r.Args.ServiceCidr,
},
}},
})
return outputs, nil
}
Expand Down
Loading

0 comments on commit d1df5b1

Please sign in to comment.