Skip to content

Commit

Permalink
Rework stack management
Browse files Browse the repository at this point in the history
- more visibility into what task manager does
  - print clear descriptions of what is being done
  - enable addition of `--dry-run` to every command
- cleaner main functions for create and delete commands
- more general abstraction for tasks
- expose all common operations through functions
- remove special purpose taks handler functions
  • Loading branch information
errordeveloper committed Apr 4, 2019
1 parent 653cd2a commit cbcb386
Show file tree
Hide file tree
Showing 12 changed files with 491 additions and 321 deletions.
71 changes: 39 additions & 32 deletions pkg/cfn/manager/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ type StackInfo struct {
Template *string
}

// ChangeSet represents a Cloudformation changeSet
// ChangeSet represents a CloudFormation ChangeSet
type ChangeSet = cloudformation.DescribeChangeSetOutput

// StackCollection stores the CloudFormation stack information
Expand Down Expand Up @@ -62,7 +62,7 @@ func NewStackCollection(provider api.ClusterProvider, spec *api.ClusterConfig) *
}
}

// DoCreateStackRequest requests the creation of a cloudformation stack.
// DoCreateStackRequest requests the creation of a CloudFormation stack
func (c *StackCollection) DoCreateStackRequest(i *Stack, templateBody []byte, tags, parameters map[string]string, withIAM bool, withNamedIAM bool) error {
input := &cloudformation.CreateStackInput{
StackName: i.StackName,
Expand Down Expand Up @@ -121,12 +121,14 @@ func (c *StackCollection) CreateStack(name string, stack builder.ResourceSet, ta
return err
}

logger.Info("deploying stack %q", name)

go c.waitUntilStackIsCreated(i, stack, errs)

return nil
}

// UpdateStack will update a cloudformation stack by creating and executing a ChangeSet.
// UpdateStack will update a CloudFormation stack by creating and executing a ChangeSet
func (c *StackCollection) UpdateStack(stackName string, changeSetName string, description string, template []byte, parameters map[string]string) error {
logger.Info(description)
i := &Stack{StackName: &stackName}
Expand Down Expand Up @@ -226,8 +228,8 @@ func defaultStackStatusFilter() []*string {
)
}

// DeleteStack kills a stack by name without waiting for DELETED status
func (c *StackCollection) DeleteStack(name string, force bool) (*Stack, error) {
// DeleteStackByName sends a request to delete the stack
func (c *StackCollection) DeleteStackByName(name string) (*Stack, error) {
i := &Stack{StackName: &name}
s, err := c.DescribeStack(i)
if err != nil {
Expand All @@ -243,38 +245,55 @@ func (c *StackCollection) DeleteStack(name string, force bool) (*Stack, error) {
}
return nil, err
}
if *s.StackStatus == cloudformation.StackStatusDeleteFailed && !force {
return nil, fmt.Errorf("stack %q previously couldn't be deleted", name)
}
i.StackId = s.StackId
return c.DeleteStackBySpec(i)
}

// WaitDeleteStackByName sends a request to delete the stack, and waits until status is DELETE_COMPLETE;
// any errors will be written to errs channel, assume completion when nil is written, do not expect
// more then one error value on the channel, it's closed immediately after it is written to
func (c *StackCollection) WaitDeleteStackByName(name string, errs chan error) error {
i, err := c.DeleteStackByName(name)
if err != nil {
return err
}

logger.Info("waiting for stack %q to get deleted", *i.StackName)

go c.waitUntilStackIsDeleted(i, errs)

return nil
}

// DeleteStackBySpec sends a request to delete the stack
func (c *StackCollection) DeleteStackBySpec(s *Stack) (*Stack, error) {
for _, tag := range s.Tags {
if *tag.Key == api.ClusterNameTag && *tag.Value == c.spec.Metadata.Name {
input := &cloudformation.DeleteStackInput{
StackName: i.StackId,
StackName: s.StackId,
}

if cfnRole := c.provider.CloudFormationRoleARN(); cfnRole != "" {
input = input.SetRoleARN(cfnRole)
}

if _, err := c.provider.CloudFormation().DeleteStack(input); err != nil {
return nil, errors.Wrapf(err, "not able to delete stack %q", name)
return nil, errors.Wrapf(err, "not able to delete stack %q", s.StackName)
}
logger.Info("will delete stack %q", name)
return i, nil
logger.Info("will delete stack %q", *s.StackName)
return s, nil
}
}

return nil, fmt.Errorf("cannot delete stack %q as it doesn't bare our %q tag", *s.StackName,
fmt.Sprintf("%s:%s", api.ClusterNameTag, c.spec.Metadata.Name))
}

// WaitDeleteStack kills a stack by name and waits for DELETED status;
// any errors will be written to errs channel, when nil is written,
// assume completion, do not expect more then one error value on the
// channel, it's closed immediately after it is written to
func (c *StackCollection) WaitDeleteStack(name string, force bool, errs chan error) error {
i, err := c.DeleteStack(name, force)
// WaitDeleteStackBySpec sends a request to delete the stack, and waits until status is DELETE_COMPLETE;
// any errors will be written to errs channel, assume completion when nil is written, do not expect
// more then one error value on the channel, it's closed immediately after it is written to
func (c *StackCollection) WaitDeleteStackBySpec(s *Stack, errs chan error) error {
i, err := c.DeleteStackBySpec(s)
if err != nil {
return err
}
Expand All @@ -286,18 +305,6 @@ func (c *StackCollection) WaitDeleteStack(name string, force bool, errs chan err
return nil
}

// BlockingWaitDeleteStack kills a stack by name and waits for DELETED status
func (c *StackCollection) BlockingWaitDeleteStack(name string, force bool) error {
i, err := c.DeleteStack(name, force)
if err != nil {
return err
}

logger.Info("waiting for stack %q to get deleted", *i.StackName)

return c.doWaitUntilStackIsDeleted(i)
}

func fmtStacksRegexForCluster(name string) string {
const ourStackRegexFmt = "^(eksctl|EKS)-%s-((cluster|nodegroup-.+)|(VPC|ServiceRole|ControlPlane|DefaultNodeGroup))$"
return fmt.Sprintf(ourStackRegexFmt, name)
Expand All @@ -315,7 +322,7 @@ func (c *StackCollection) DescribeStacks() ([]*Stack, error) {
return stacks, nil
}

// DescribeStackEvents describes the occurred stack events
// DescribeStackEvents describes the events that have occurred on the stack
func (c *StackCollection) DescribeStackEvents(i *Stack) ([]*cloudformation.StackEvent, error) {
input := &cloudformation.DescribeStackEventsInput{
StackName: i.StackName,
Expand Down Expand Up @@ -410,7 +417,7 @@ func (c *StackCollection) doExecuteChangeSet(stackName string, changeSetName str
return nil
}

// DescribeStackChangeSet gets a cloudformation changeset.
// DescribeStackChangeSet describes a ChangeSet by name
func (c *StackCollection) DescribeStackChangeSet(i *Stack, changeSetName string) (*ChangeSet, error) {
input := &cloudformation.DescribeChangeSetInput{
StackName: i.StackName,
Expand Down
17 changes: 3 additions & 14 deletions pkg/cfn/manager/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@ func (c *StackCollection) makeClusterStackName() string {
return "eksctl-" + c.spec.Metadata.Name + "-cluster"
}

// CreateCluster creates the cluster
func (c *StackCollection) CreateCluster(errs chan error, _ interface{}) error {
// createClusterTask creates the cluster
func (c *StackCollection) createClusterTask(errs chan error) error {
name := c.makeClusterStackName()
logger.Info("building cluster stack %q", name)
stack := builder.NewClusterResourceSet(c.provider, c.spec)
if err := stack.AddAllResources(); err != nil {
return err
}

// Unlike with `CreateNodeGroup`, all tags are already set for the cluster stack
// Unlike with `createNodeGroupTask`, all tags are already set for the cluster stack
return c.CreateStack(name, stack, nil, nil, errs)
}

Expand All @@ -56,17 +56,6 @@ func (c *StackCollection) DescribeClusterStack() (*Stack, error) {
return nil, nil
}

// DeleteCluster deletes the cluster
func (c *StackCollection) DeleteCluster(force bool) error {
_, err := c.DeleteStack(c.makeClusterStackName(), force)
return err
}

// WaitDeleteCluster waits till the cluster is deleted
func (c *StackCollection) WaitDeleteCluster(force bool) error {
return c.BlockingWaitDeleteStack(c.makeClusterStackName(), force)
}

// AppendNewClusterStackResource will update cluster
// stack with new resources in append-only way
func (c *StackCollection) AppendNewClusterStackResource(dryRun bool) (bool, error) {
Expand Down
49 changes: 49 additions & 0 deletions pkg/cfn/manager/create_tasks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package manager

import (
"fmt"

"k8s.io/apimachinery/pkg/util/sets"
)

// CreateTasksForClusterWithNodeGroups defines all tasks required to create a cluster along
// with some nodegroups; see CreateAllNodeGroups for how onlyNodeGroupSubset works
func (c *StackCollection) CreateTasksForClusterWithNodeGroups(onlyNodeGroupSubset sets.String) *TaskSet {
tasks := &TaskSet{Parallel: false}

tasks.Append(
&taskWithoutParams{
info: fmt.Sprintf("create cluster control plane %q", c.spec.Metadata.Name),
call: c.createClusterTask,
},
)

nodeGroupTasks := c.CreateTasksForNodeGroups(onlyNodeGroupSubset)
if nodeGroupTasks.Len() > 0 {
nodeGroupTasks.Sub = true
tasks.Append(nodeGroupTasks)
}

return tasks
}

// CreateTasksForNodeGroups defines tasks required to create all of the nodegroups if
// onlySubset is nil, otherwise just the tasks for nodegroups that are in onlySubset
// will be defined
func (c *StackCollection) CreateTasksForNodeGroups(onlySubset sets.String) *TaskSet {
tasks := &TaskSet{Parallel: true}

for i := range c.spec.NodeGroups {
ng := c.spec.NodeGroups[i]
if onlySubset != nil && !onlySubset.Has(ng.Name) {
continue
}
tasks.Append(&taskWithNodeGroupSpec{
info: fmt.Sprintf("create nodegroup %q", ng.Name),
nodeGroup: ng,
call: c.createNodeGroupTask,
})
}

return tasks
}
87 changes: 87 additions & 0 deletions pkg/cfn/manager/delete_tasks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package manager

import (
"fmt"

"github.com/aws/aws-sdk-go/service/cloudformation"

"k8s.io/apimachinery/pkg/util/sets"
)

// DeleteTasksForClusterWithNodeGroups defines tasks required to delete all the nodegroup
// stacks and the cluster
func (c *StackCollection) DeleteTasksForClusterWithNodeGroups(wait bool, cleanup func(chan error, string) error) (*TaskSet, error) {
tasks := &TaskSet{Parallel: false}

nodeGroupTasks, err := c.DeleteTasksForNodeGroups(nil, true, cleanup)
if err != nil {
return nil, err
}
if nodeGroupTasks.Len() > 0 {
nodeGroupTasks.Sub = true
tasks.Append(nodeGroupTasks)
}

clusterStack, err := c.DescribeClusterStack()
if err != nil {
return nil, err
}

info := fmt.Sprintf("delete cluster control plane %q", c.spec.Metadata.Name)
if wait {
tasks.Append(&taskWithStackSpec{
info: info,
stack: clusterStack,
call: c.WaitDeleteStackBySpec,
})
} else {
tasks.Append(&asyncTaskWithStackSpec{
info: info,
stack: clusterStack,
call: c.DeleteStackBySpec,
})
}

return tasks, nil
}

// DeleteTasksForNodeGroups defines tasks required to delete all of the nodegroups if
// onlySubset is nil, otherwise just the tasks for nodegroups that are in onlySubset
// will be defined
func (c *StackCollection) DeleteTasksForNodeGroups(onlySubset sets.String, wait bool, cleanup func(chan error, string) error) (*TaskSet, error) {
nodeGroupStacks, err := c.DescribeNodeGroupStacks()
if err != nil {
return nil, err
}

tasks := &TaskSet{Parallel: true}

for _, s := range nodeGroupStacks {
name := getNodeGroupName(s)
if onlySubset != nil && !onlySubset.Has(name) {
continue
}
if *s.StackStatus == cloudformation.StackStatusDeleteFailed && cleanup != nil {
tasks.Append(&taskWithNameParam{
info: fmt.Sprintf("cleanup for nodegroup %q", name),
call: cleanup,
})
}
info := fmt.Sprintf("delete nodegroup %q", name)
if wait {
tasks.Append(&taskWithStackSpec{
info: info,
stack: s,
call: c.WaitDeleteStackBySpec,
})
} else {
tasks.Append(&asyncTaskWithStackSpec{
info: info,
stack: s,
call: c.DeleteStackBySpec,
})
}
}

return tasks, nil
}
Loading

0 comments on commit cbcb386

Please sign in to comment.