From f79ece8243ef53e02ad7731fce6ee224c95c50ed Mon Sep 17 00:00:00 2001 From: Luay-Sol <93928179+Luay-Sol@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:43:34 -0500 Subject: [PATCH] feat: Add project depends on functionality (#3292) * feat: implemented the code for the depends on functionality Co-authored-by: Vincent <106497818+vincentgna@users.noreply.github.com> --- .../docs/repo-level-atlantis-yaml.md | 42 ++++++ server/core/config/raw/project.go | 8 ++ server/core/config/valid/global_cfg.go | 5 +- server/core/config/valid/repo_cfg.go | 1 + server/events/command/project_context.go | 10 ++ server/events/command_requirement_handler.go | 18 +++ .../command_requirement_handler_test.go | 126 ++++++++++++++++++ .../mocks/mock_command_requirement_handler.go | 19 +++ .../events/project_command_context_builder.go | 9 +- server/events/project_command_runner.go | 5 + 10 files changed, 238 insertions(+), 5 deletions(-) diff --git a/runatlantis.io/docs/repo-level-atlantis-yaml.md b/runatlantis.io/docs/repo-level-atlantis-yaml.md index 777552c634..c4c6ed3792 100644 --- a/runatlantis.io/docs/repo-level-atlantis-yaml.md +++ b/runatlantis.io/docs/repo-level-atlantis-yaml.md @@ -75,6 +75,8 @@ projects: apply_requirements: [mergeable, approved, undiverged] import_requirements: [mergeable, approved, undiverged] execution_order_group: 1 + depends_on: + - project-1 workflow: myworkflow workflows: myworkflow: @@ -289,6 +291,46 @@ in each group one by one. If any plan/apply fails and `abort_on_execution_order_fail` is set to true on a repo level, all the following groups will be aborted. For this example, if project2 fails then project1 will not run. +Execution order groups are useful when you have dependencies between projects. However, they are only applicable in the case where +you initiate a global apply for all of your projects, i.e `atlantis apply`. If you initiate an apply on a single project, then the execution order groups are ignored. +Thus, the `depends_on` key is more useful in this case. and can be used in conjunction with execution order groups. + +The following configuration is an example of how to use execution order groups and depends_on together to enforce dependencies between projects. +```yaml +version: 3 +projects: +- name: development + dir: . + autoplan: + when_modified: ["*.tf", "vars/development.tfvars"] + execution_order_group: 1 + workspace: development + workflow: infra +- name: staging + dir: . + autoplan: + when_modified: ["*.tf", "vars/staging.tfvars"] + depends_on: ["development"] + execution_order_group: 2 + workspace: staging + workflow: infra +- name: production + dir: . + autoplan: + when_modified: ["*.tf", "vars/production.tfvars"] + depends_on: ["staging"] + execution_order_group: 3 + workspace: production + workflow: infra +``` +the `depends_on` feature will make sure that `production` is not applied before `staging` for example. + +::: tip +What Happens if one or more project's dependencies are not applied? + +If there's one or more projects in the dependency list which is not in applied status, users will see an error message like this: +`Can't apply your project unless you apply its dependencies` +::: ### Autodiscovery Config ```yaml autodiscover: diff --git a/server/core/config/raw/project.go b/server/core/config/raw/project.go index e288818a9e..d73062cef3 100644 --- a/server/core/config/raw/project.go +++ b/server/core/config/raw/project.go @@ -31,6 +31,7 @@ type Project struct { PlanRequirements []string `yaml:"plan_requirements,omitempty"` ApplyRequirements []string `yaml:"apply_requirements,omitempty"` ImportRequirements []string `yaml:"import_requirements,omitempty"` + DependsOn []string `yaml:"depends_on,omitempty"` DeleteSourceBranchOnMerge *bool `yaml:"delete_source_branch_on_merge,omitempty"` RepoLocking *bool `yaml:"repo_locking,omitempty"` ExecutionOrderGroup *int `yaml:"execution_order_group,omitempty"` @@ -74,12 +75,17 @@ func (p Project) Validate() error { return errors.Wrapf(err, "parsing: %s", branch) } + DependsOn := func(value interface{}) error { + return nil + } + return validation.ValidateStruct(&p, validation.Field(&p.Dir, validation.Required, validation.By(hasDotDot)), validation.Field(&p.PlanRequirements, validation.By(validPlanReq)), validation.Field(&p.ApplyRequirements, validation.By(validApplyReq)), validation.Field(&p.ImportRequirements, validation.By(validImportReq)), validation.Field(&p.TerraformVersion, validation.By(VersionValidator)), + validation.Field(&p.DependsOn, validation.By(DependsOn)), validation.Field(&p.Name, validation.By(validName)), validation.Field(&p.Branch, validation.By(branchValid)), ) @@ -123,6 +129,8 @@ func (p Project) ToValid() valid.Project { v.Name = p.Name + v.DependsOn = p.DependsOn + if p.DeleteSourceBranchOnMerge != nil { v.DeleteSourceBranchOnMerge = p.DeleteSourceBranchOnMerge } diff --git a/server/core/config/valid/global_cfg.go b/server/core/config/valid/global_cfg.go index 0006e937c5..3c43e59df0 100644 --- a/server/core/config/valid/global_cfg.go +++ b/server/core/config/valid/global_cfg.go @@ -17,10 +17,7 @@ const PoliciesPassedCommandReq = "policies_passed" const PlanRequirementsKey = "plan_requirements" const ApplyRequirementsKey = "apply_requirements" const ImportRequirementsKey = "import_requirements" -const PreWorkflowHooksKey = "pre_workflow_hooks" const WorkflowKey = "workflow" -const PostWorkflowHooksKey = "post_workflow_hooks" -const AllowedWorkflowsKey = "allowed_workflows" const AllowedOverridesKey = "allowed_overrides" const AllowCustomWorkflowsKey = "allow_custom_workflows" const DefaultWorkflowName = "default" @@ -94,6 +91,7 @@ type MergedProjectCfg struct { ImportRequirements []string Workflow Workflow AllowedWorkflows []string + DependsOn []string RepoRelDir string Workspace string Name string @@ -394,6 +392,7 @@ func (g GlobalCfg) MergeProjectCfg(log logging.SimpleLogging, repoID string, pro Workflow: workflow, RepoRelDir: proj.Dir, Workspace: proj.Workspace, + DependsOn: proj.DependsOn, Name: proj.GetName(), AutoplanEnabled: proj.Autoplan.Enabled, TerraformVersion: proj.TerraformVersion, diff --git a/server/core/config/valid/repo_cfg.go b/server/core/config/valid/repo_cfg.go index f0aed102ad..95e36b1f27 100644 --- a/server/core/config/valid/repo_cfg.go +++ b/server/core/config/valid/repo_cfg.go @@ -151,6 +151,7 @@ type Project struct { PlanRequirements []string ApplyRequirements []string ImportRequirements []string + DependsOn []string DeleteSourceBranchOnMerge *bool RepoLocking *bool ExecutionOrderGroup int diff --git a/server/events/command/project_context.go b/server/events/command/project_context.go index 1e2521e38c..c06681ef82 100644 --- a/server/events/command/project_context.go +++ b/server/events/command/project_context.go @@ -57,6 +57,13 @@ type ProjectContext struct { // If the pull request branch is from the same repository then HeadRepo will // be the same as BaseRepo. HeadRepo models.Repo + // Dependencies are a list of project that this project relies on + // their apply status. These projects must be applied first. + // + // Atlantis uses this information to valid the apply + // orders and to warn the user if they're applying a project that + // depends on other projects. + DependsOn []string // Log is a logger that's been set up for this context. Log logging.SimpleLogging // Scope is the scope for reporting stats setup for this context @@ -65,8 +72,11 @@ type ProjectContext struct { PullReqStatus models.PullReqStatus // CurrentProjectPlanStatus is the status of the current project prior to this command. ProjectPlanStatus models.ProjectPlanStatus + //PullStatus is the status of the current pull request prior to this command. + PullStatus *models.PullStatus // ProjectPolicyStatus is the status of policy sets of the current project prior to this command. ProjectPolicyStatus []models.PolicySetStatus + // Pull is the pull request we're responding to. Pull models.PullRequest // ProjectName is the name of the project set in atlantis.yaml. If there was diff --git a/server/events/command_requirement_handler.go b/server/events/command_requirement_handler.go index 8af12bec54..5c7b1c1d54 100644 --- a/server/events/command_requirement_handler.go +++ b/server/events/command_requirement_handler.go @@ -1,13 +1,17 @@ package events import ( + "fmt" + "github.com/runatlantis/atlantis/server/core/config/raw" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/events/command" + "github.com/runatlantis/atlantis/server/events/models" ) //go:generate pegomock generate --package mocks -o mocks/mock_command_requirement_handler.go CommandRequirementHandler type CommandRequirementHandler interface { + ValidateProjectDependencies(ctx command.ProjectContext) (string, error) ValidatePlanProject(repoDir string, ctx command.ProjectContext) (string, error) ValidateApplyProject(repoDir string, ctx command.ProjectContext) (string, error) ValidateImportProject(repoDir string, ctx command.ProjectContext) (string, error) @@ -65,6 +69,20 @@ func (a *DefaultCommandRequirementHandler) ValidateApplyProject(repoDir string, return "", nil } +func (a *DefaultCommandRequirementHandler) ValidateProjectDependencies(ctx command.ProjectContext) (failure string, err error) { + for _, dependOnProject := range ctx.DependsOn { + + for _, project := range ctx.PullStatus.Projects { + + if project.ProjectName == dependOnProject && project.Status != models.AppliedPlanStatus && project.Status != models.PlannedNoChangesPlanStatus { + return fmt.Sprintf("Can't apply your project unless you apply its dependencies: [%s]", project.ProjectName), nil + } + } + } + + return "", nil +} + func (a *DefaultCommandRequirementHandler) ValidateImportProject(repoDir string, ctx command.ProjectContext) (failure string, err error) { for _, req := range ctx.ImportRequirements { switch req { diff --git a/server/events/command_requirement_handler_test.go b/server/events/command_requirement_handler_test.go index 7a9891b07c..1c737f05aa 100644 --- a/server/events/command_requirement_handler_test.go +++ b/server/events/command_requirement_handler_test.go @@ -207,6 +207,132 @@ func TestAggregateApplyRequirements_ValidateApplyProject(t *testing.T) { } } +func TestRequirements_ValidateProjectDependencies(t *testing.T) { + tests := []struct { + name string + ctx command.ProjectContext + setup func(workingDir *mocks.MockWorkingDir) + wantFailure string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "pass no dependencies", + ctx: command.ProjectContext{}, + wantErr: assert.NoError, + }, + { + name: "pass all dependencies applied", + ctx: command.ProjectContext{ + DependsOn: []string{"project1"}, + PullStatus: &models.PullStatus{ + Projects: []models.ProjectStatus{ + { + ProjectName: "project1", + Status: models.AppliedPlanStatus, + }, + }, + }, + }, + wantErr: assert.NoError, + }, + { + name: "Fail all dependencies are not applied", + ctx: command.ProjectContext{ + DependsOn: []string{"project1", "project2"}, + PullStatus: &models.PullStatus{ + Projects: []models.ProjectStatus{ + { + ProjectName: "project1", + Status: models.PlannedPlanStatus, + }, + { + ProjectName: "project2", + Status: models.ErroredApplyStatus, + }, + }, + }, + }, + wantFailure: "Can't apply your project unless you apply its dependencies: [project1]", + wantErr: assert.NoError, + }, + { + name: "Fail one of dependencies is not applied", + ctx: command.ProjectContext{ + DependsOn: []string{"project1", "project2"}, + PullStatus: &models.PullStatus{ + Projects: []models.ProjectStatus{ + { + ProjectName: "project1", + Status: models.AppliedPlanStatus, + }, + { + ProjectName: "project2", + Status: models.ErroredApplyStatus, + }, + }, + }, + }, + wantFailure: "Can't apply your project unless you apply its dependencies: [project2]", + wantErr: assert.NoError, + }, + { + name: "Should not fail if one of dependencies is not applied but it has no changes to apply", + ctx: command.ProjectContext{ + DependsOn: []string{"project1", "project2"}, + PullStatus: &models.PullStatus{ + Projects: []models.ProjectStatus{ + { + ProjectName: "project1", + Status: models.AppliedPlanStatus, + }, + { + ProjectName: "project2", + Status: models.PlannedNoChangesPlanStatus, + }, + }, + }, + }, + wantErr: assert.NoError, + }, + { + name: "In the case of more than one dependency, should not continue to check dependencies if one of them is not in applied status", + ctx: command.ProjectContext{ + DependsOn: []string{"project1", "project2"}, + PullStatus: &models.PullStatus{ + Projects: []models.ProjectStatus{ + { + ProjectName: "project1", + Status: models.AppliedPlanStatus, + }, + { + ProjectName: "project2", + Status: models.ErroredApplyStatus, + }, + { + ProjectName: "project3", + Status: models.PlannedPlanStatus, + }, + }, + }, + }, + wantFailure: "Can't apply your project unless you apply its dependencies: [project2]", + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + RegisterMockTestingT(t) + workingDir := mocks.NewMockWorkingDir() + a := &events.DefaultCommandRequirementHandler{WorkingDir: workingDir} + gotFailure, err := a.ValidateProjectDependencies(tt.ctx) + if !tt.wantErr(t, err, fmt.Sprintf("ValidateProjectDependencies(%v)", tt.ctx)) { + return + } + assert.Equalf(t, tt.wantFailure, gotFailure, "ValidateProjectDependencies(%v)", tt.ctx) + }) + } +} + func TestAggregateApplyRequirements_ValidateImportProject(t *testing.T) { repoDir := "repoDir" fullRequirements := []string{ diff --git a/server/events/mocks/mock_command_requirement_handler.go b/server/events/mocks/mock_command_requirement_handler.go index 8b6dd3c775..d302bf4525 100644 --- a/server/events/mocks/mock_command_requirement_handler.go +++ b/server/events/mocks/mock_command_requirement_handler.go @@ -44,6 +44,25 @@ func (mock *MockCommandRequirementHandler) ValidateApplyProject(repoDir string, return ret0, ret1 } +func (mock *MockCommandRequirementHandler) ValidateProjectDependencies(_param0 command.ProjectContext) (string, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().") + } + params := []pegomock.Param{_param0} + result := pegomock.GetGenericMockFrom(mock).Invoke("ValidateProjectDependencies", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 string + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(string) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + func (mock *MockCommandRequirementHandler) ValidateImportProject(repoDir string, ctx command.ProjectContext) (string, error) { if mock == nil { panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().") diff --git a/server/events/project_command_context_builder.go b/server/events/project_command_context_builder.go index 0b4bf00f29..5ed6dad94c 100644 --- a/server/events/project_command_context_builder.go +++ b/server/events/project_command_context_builder.go @@ -144,6 +144,7 @@ func (cb *DefaultProjectCommandContextBuilder) BuildProjectContext( abortOnExcecutionOrderFail, ctx.Scope, ctx.PullRequestStatus, + ctx.PullStatus, ) projectCmds = append(projectCmds, projectCmdContext) @@ -215,6 +216,7 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext( abortOnExcecutionOrderFail, ctx.Scope, ctx.PullRequestStatus, + ctx.PullStatus, )) } @@ -238,7 +240,8 @@ func newProjectCommandContext(ctx *command.Context, verbose bool, abortOnExcecutionOrderFail bool, scope tally.Scope, - pullStatus models.PullReqStatus, + pullReqStatus models.PullReqStatus, + pullStatus *models.PullStatus, ) command.ProjectContext { var projectPlanStatus models.ProjectPlanStatus @@ -275,6 +278,7 @@ func newProjectCommandContext(ctx *command.Context, ParallelApplyEnabled: parallelApplyEnabled, ParallelPlanEnabled: parallelPlanEnabled, ParallelPolicyCheckEnabled: parallelPlanEnabled, + DependsOn: projCfg.DependsOn, AutoplanEnabled: projCfg.AutoplanEnabled, Steps: steps, HeadRepo: ctx.HeadRepo, @@ -297,7 +301,8 @@ func newProjectCommandContext(ctx *command.Context, PolicySets: policySets, PolicySetTarget: ctx.PolicySet, ClearPolicyApproval: ctx.ClearPolicyApproval, - PullReqStatus: pullStatus, + PullReqStatus: pullReqStatus, + PullStatus: pullStatus, JobID: uuid.New().String(), ExecutionOrderGroup: projCfg.ExecutionOrderGroup, AbortOnExcecutionOrderFail: abortOnExcecutionOrderFail, diff --git a/server/events/project_command_runner.go b/server/events/project_command_runner.go index b67b77065b..736b8ac31b 100644 --- a/server/events/project_command_runner.go +++ b/server/events/project_command_runner.go @@ -607,6 +607,11 @@ func (p *DefaultProjectCommandRunner) doApply(ctx command.ProjectContext) (apply return "", failure, err } + failure, err = p.CommandRequirementHandler.ValidateProjectDependencies(ctx) + if failure != "" || err != nil { + return "", failure, err + } + // Acquire internal lock for the directory we're going to operate in. unlockFn, err := p.WorkingDirLocker.TryLock(ctx.Pull.BaseRepo.FullName, ctx.Pull.Num, ctx.Workspace, ctx.RepoRelDir) if err != nil {