feat: Add project depends on functionality (#3292)

* feat: implemented the code for the depends on functionality

Co-authored-by: Vincent <106497818+vincentgna@users.noreply.github.com>

runatlantis.io/docs/repo-level-atlantis-yaml.md

@@ -75,6 +75,8 @@ projects:
   apply_requirements: [mergeable, approved, undiverged]
   import_requirements: [mergeable, approved, undiverged]
   execution_order_group: 1
+  depends_on:
+    - project-1
   workflow: myworkflow
 workflows:
   myworkflow:
@@ -289,6 +291,46 @@ in each group one by one.
 If any plan/apply fails and `abort_on_execution_order_fail` is set to true on a repo level, all the 
 following groups will be aborted. For this example, if project2 fails then project1 will not run.
 
+Execution order groups are useful when you have dependencies between projects. However, they are only applicable in the case where
+you initiate a global apply for all of your projects, i.e `atlantis apply`. If you initiate an apply on a single project, then the execution order groups are ignored.
+Thus, the `depends_on` key is more useful in this case. and can be used in conjunction with execution order groups.
+
+The following configuration is an example of how to use execution order groups and depends_on together to enforce dependencies between projects.
+```yaml
+version: 3
+projects:
+- name: development
+  dir: .
+  autoplan:
+    when_modified: ["*.tf", "vars/development.tfvars"]
+  execution_order_group: 1
+  workspace: development
+  workflow: infra
+- name: staging
+  dir: .
+  autoplan:
+    when_modified: ["*.tf", "vars/staging.tfvars"]
+  depends_on: ["development"]
+  execution_order_group: 2
+  workspace: staging
+  workflow: infra
+- name: production
+  dir: .
+  autoplan:
+    when_modified: ["*.tf", "vars/production.tfvars"]
+  depends_on: ["staging"]
+  execution_order_group: 3
+  workspace: production
+  workflow: infra
+```
+the `depends_on` feature will make sure that `production` is not applied before `staging` for example.
+
+::: tip
+What Happens if one or more project's dependencies are not applied?
+
+If there's one or more projects in the dependency list which is not in applied status, users will see an error message like this:
+`Can't apply your project unless you apply its dependencies`
+:::
 ### Autodiscovery Config
 ```yaml
 autodiscover:

server/core/config/raw/project.go

@@ -31,6 +31,7 @@ type Project struct {
 	PlanRequirements          []string  `yaml:"plan_requirements,omitempty"`
 	ApplyRequirements         []string  `yaml:"apply_requirements,omitempty"`
 	ImportRequirements        []string  `yaml:"import_requirements,omitempty"`
+	DependsOn                 []string  `yaml:"depends_on,omitempty"`
 	DeleteSourceBranchOnMerge *bool     `yaml:"delete_source_branch_on_merge,omitempty"`
 	RepoLocking               *bool     `yaml:"repo_locking,omitempty"`
 	ExecutionOrderGroup       *int      `yaml:"execution_order_group,omitempty"`
@@ -74,12 +75,17 @@ func (p Project) Validate() error {
 		return errors.Wrapf(err, "parsing: %s", branch)
 	}
 
+	DependsOn := func(value interface{}) error {
+		return nil
+	}
+
 	return validation.ValidateStruct(&p,
 		validation.Field(&p.Dir, validation.Required, validation.By(hasDotDot)),
 		validation.Field(&p.PlanRequirements, validation.By(validPlanReq)),
 		validation.Field(&p.ApplyRequirements, validation.By(validApplyReq)),
 		validation.Field(&p.ImportRequirements, validation.By(validImportReq)),
 		validation.Field(&p.TerraformVersion, validation.By(VersionValidator)),
+		validation.Field(&p.DependsOn, validation.By(DependsOn)),
 		validation.Field(&p.Name, validation.By(validName)),
 		validation.Field(&p.Branch, validation.By(branchValid)),
 	)
@@ -123,6 +129,8 @@ func (p Project) ToValid() valid.Project {
 
 	v.Name = p.Name
 
+	v.DependsOn = p.DependsOn
+
 	if p.DeleteSourceBranchOnMerge != nil {
 		v.DeleteSourceBranchOnMerge = p.DeleteSourceBranchOnMerge
 	}

server/core/config/valid/global_cfg.go

@@ -17,10 +17,7 @@ const PoliciesPassedCommandReq = "policies_passed"
 const PlanRequirementsKey = "plan_requirements"
 const ApplyRequirementsKey = "apply_requirements"
 const ImportRequirementsKey = "import_requirements"
-const PreWorkflowHooksKey = "pre_workflow_hooks"
 const WorkflowKey = "workflow"
-const PostWorkflowHooksKey = "post_workflow_hooks"
-const AllowedWorkflowsKey = "allowed_workflows"
 const AllowedOverridesKey = "allowed_overrides"
 const AllowCustomWorkflowsKey = "allow_custom_workflows"
 const DefaultWorkflowName = "default"
@@ -94,6 +91,7 @@ type MergedProjectCfg struct {
 	ImportRequirements        []string
 	Workflow                  Workflow
 	AllowedWorkflows          []string
+	DependsOn                 []string
 	RepoRelDir                string
 	Workspace                 string
 	Name                      string
@@ -394,6 +392,7 @@ func (g GlobalCfg) MergeProjectCfg(log logging.SimpleLogging, repoID string, pro
 		Workflow:                  workflow,
 		RepoRelDir:                proj.Dir,
 		Workspace:                 proj.Workspace,
+		DependsOn:                 proj.DependsOn,
 		Name:                      proj.GetName(),
 		AutoplanEnabled:           proj.Autoplan.Enabled,
 		TerraformVersion:          proj.TerraformVersion,

server/core/config/valid/repo_cfg.go

@@ -151,6 +151,7 @@ type Project struct {
 	PlanRequirements          []string
 	ApplyRequirements         []string
 	ImportRequirements        []string
+	DependsOn                 []string
 	DeleteSourceBranchOnMerge *bool
 	RepoLocking               *bool
 	ExecutionOrderGroup       int

server/events/command/project_context.go

@@ -57,6 +57,13 @@ type ProjectContext struct {
 	// If the pull request branch is from the same repository then HeadRepo will
 	// be the same as BaseRepo.
 	HeadRepo models.Repo
+	// Dependencies are a list of project that this project relies on
+	// their apply status. These projects must be applied first.
+	//
+	// Atlantis uses this information to valid the apply
+	// orders and to warn the user if they're applying a project that
+	// depends on other projects.
+	DependsOn []string
 	// Log is a logger that's been set up for this context.
 	Log logging.SimpleLogging
 	// Scope is the scope for reporting stats setup for this context
@@ -65,8 +72,11 @@ type ProjectContext struct {
 	PullReqStatus models.PullReqStatus
 	// CurrentProjectPlanStatus is the status of the current project prior to this command.
 	ProjectPlanStatus models.ProjectPlanStatus
+	//PullStatus is the status of the current pull request prior to this command.
+	PullStatus *models.PullStatus
 	// ProjectPolicyStatus is the status of policy sets of the current project prior to this command.
 	ProjectPolicyStatus []models.PolicySetStatus
+
 	// Pull is the pull request we're responding to.
 	Pull models.PullRequest
 	// ProjectName is the name of the project set in atlantis.yaml. If there was

server/events/command_requirement_handler.go

@@ -1,13 +1,17 @@
 package events
 
 import (
+	"fmt"
+
 	"github.com/runatlantis/atlantis/server/core/config/raw"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 	"github.com/runatlantis/atlantis/server/events/command"
+	"github.com/runatlantis/atlantis/server/events/models"
 )
 
 //go:generate pegomock generate --package mocks -o mocks/mock_command_requirement_handler.go CommandRequirementHandler
 type CommandRequirementHandler interface {
+	ValidateProjectDependencies(ctx command.ProjectContext) (string, error)
 	ValidatePlanProject(repoDir string, ctx command.ProjectContext) (string, error)
 	ValidateApplyProject(repoDir string, ctx command.ProjectContext) (string, error)
 	ValidateImportProject(repoDir string, ctx command.ProjectContext) (string, error)
@@ -65,6 +69,20 @@ func (a *DefaultCommandRequirementHandler) ValidateApplyProject(repoDir string,
 	return "", nil
 }
 
+func (a *DefaultCommandRequirementHandler) ValidateProjectDependencies(ctx command.ProjectContext) (failure string, err error) {
+	for _, dependOnProject := range ctx.DependsOn {
+
+		for _, project := range ctx.PullStatus.Projects {
+
+			if project.ProjectName == dependOnProject && project.Status != models.AppliedPlanStatus && project.Status != models.PlannedNoChangesPlanStatus {
+				return fmt.Sprintf("Can't apply your project unless you apply its dependencies: [%s]", project.ProjectName), nil
+			}
+		}
+	}
+
+	return "", nil
+}
+
 func (a *DefaultCommandRequirementHandler) ValidateImportProject(repoDir string, ctx command.ProjectContext) (failure string, err error) {
 	for _, req := range ctx.ImportRequirements {
 		switch req {

server/events/command_requirement_handler_test.go

@@ -207,6 +207,132 @@ func TestAggregateApplyRequirements_ValidateApplyProject(t *testing.T) {
 	}
 }
 
+func TestRequirements_ValidateProjectDependencies(t *testing.T) {
+	tests := []struct {
+		name        string
+		ctx         command.ProjectContext
+		setup       func(workingDir *mocks.MockWorkingDir)
+		wantFailure string
+		wantErr     assert.ErrorAssertionFunc
+	}{
+		{
+			name:    "pass no dependencies",
+			ctx:     command.ProjectContext{},
+			wantErr: assert.NoError,
+		},
+		{
+			name: "pass all dependencies applied",
+			ctx: command.ProjectContext{
+				DependsOn: []string{"project1"},
+				PullStatus: &models.PullStatus{
+					Projects: []models.ProjectStatus{
+						{
+							ProjectName: "project1",
+							Status:      models.AppliedPlanStatus,
+						},
+					},
+				},
+			},
+			wantErr: assert.NoError,
+		},
+		{
+			name: "Fail all dependencies are not applied",
+			ctx: command.ProjectContext{
+				DependsOn: []string{"project1", "project2"},
+				PullStatus: &models.PullStatus{
+					Projects: []models.ProjectStatus{
+						{
+							ProjectName: "project1",
+							Status:      models.PlannedPlanStatus,
+						},
+						{
+							ProjectName: "project2",
+							Status:      models.ErroredApplyStatus,
+						},
+					},
+				},
+			},
+			wantFailure: "Can't apply your project unless you apply its dependencies: [project1]",
+			wantErr:     assert.NoError,
+		},
+		{
+			name: "Fail one of dependencies is not applied",
+			ctx: command.ProjectContext{
+				DependsOn: []string{"project1", "project2"},
+				PullStatus: &models.PullStatus{
+					Projects: []models.ProjectStatus{
+						{
+							ProjectName: "project1",
+							Status:      models.AppliedPlanStatus,
+						},
+						{
+							ProjectName: "project2",
+							Status:      models.ErroredApplyStatus,
+						},
+					},
+				},
+			},
+			wantFailure: "Can't apply your project unless you apply its dependencies: [project2]",
+			wantErr:     assert.NoError,
+		},
+		{
+			name: "Should not fail if one of dependencies is not applied but it has no changes to apply",
+			ctx: command.ProjectContext{
+				DependsOn: []string{"project1", "project2"},
+				PullStatus: &models.PullStatus{
+					Projects: []models.ProjectStatus{
+						{
+							ProjectName: "project1",
+							Status:      models.AppliedPlanStatus,
+						},
+						{
+							ProjectName: "project2",
+							Status:      models.PlannedNoChangesPlanStatus,
+						},
+					},
+				},
+			},
+			wantErr: assert.NoError,
+		},
+		{
+			name: "In the case of more than one dependency, should not continue to check dependencies if one of them is not in applied status",
+			ctx: command.ProjectContext{
+				DependsOn: []string{"project1", "project2"},
+				PullStatus: &models.PullStatus{
+					Projects: []models.ProjectStatus{
+						{
+							ProjectName: "project1",
+							Status:      models.AppliedPlanStatus,
+						},
+						{
+							ProjectName: "project2",
+							Status:      models.ErroredApplyStatus,
+						},
+						{
+							ProjectName: "project3",
+							Status:      models.PlannedPlanStatus,
+						},
+					},
+				},
+			},
+			wantFailure: "Can't apply your project unless you apply its dependencies: [project2]",
+			wantErr:     assert.NoError,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			RegisterMockTestingT(t)
+			workingDir := mocks.NewMockWorkingDir()
+			a := &events.DefaultCommandRequirementHandler{WorkingDir: workingDir}
+			gotFailure, err := a.ValidateProjectDependencies(tt.ctx)
+			if !tt.wantErr(t, err, fmt.Sprintf("ValidateProjectDependencies(%v)", tt.ctx)) {
+				return
+			}
+			assert.Equalf(t, tt.wantFailure, gotFailure, "ValidateProjectDependencies(%v)", tt.ctx)
+		})
+	}
+}
+
 func TestAggregateApplyRequirements_ValidateImportProject(t *testing.T) {
 	repoDir := "repoDir"
 	fullRequirements := []string{

server/events/mocks/mock_command_requirement_handler.go

@@ -44,6 +44,25 @@ func (mock *MockCommandRequirementHandler) ValidateApplyProject(repoDir string,
 	return ret0, ret1
 }
 
+func (mock *MockCommandRequirementHandler) ValidateProjectDependencies(_param0 command.ProjectContext) (string, error) {
+	if mock == nil {
+		panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().")
+	}
+	params := []pegomock.Param{_param0}
+	result := pegomock.GetGenericMockFrom(mock).Invoke("ValidateProjectDependencies", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()})
+	var ret0 string
+	var ret1 error
+	if len(result) != 0 {
+		if result[0] != nil {
+			ret0 = result[0].(string)
+		}
+		if result[1] != nil {
+			ret1 = result[1].(error)
+		}
+	}
+	return ret0, ret1
+}
+
 func (mock *MockCommandRequirementHandler) ValidateImportProject(repoDir string, ctx command.ProjectContext) (string, error) {
 	if mock == nil {
 		panic("mock must not be nil. Use myMock := NewMockCommandRequirementHandler().")

server/events/project_command_context_builder.go

@@ -144,6 +144,7 @@ func (cb *DefaultProjectCommandContextBuilder) BuildProjectContext(
 		abortOnExcecutionOrderFail,
 		ctx.Scope,
 		ctx.PullRequestStatus,
+		ctx.PullStatus,
 	)
 
 	projectCmds = append(projectCmds, projectCmdContext)
@@ -215,6 +216,7 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext(
 			abortOnExcecutionOrderFail,
 			ctx.Scope,
 			ctx.PullRequestStatus,
+			ctx.PullStatus,
 		))
 	}
 
@@ -238,7 +240,8 @@ func newProjectCommandContext(ctx *command.Context,
 	verbose bool,
 	abortOnExcecutionOrderFail bool,
 	scope tally.Scope,
-	pullStatus models.PullReqStatus,
+	pullReqStatus models.PullReqStatus,
+	pullStatus *models.PullStatus,
 ) command.ProjectContext {
 
 	var projectPlanStatus models.ProjectPlanStatus
@@ -275,6 +278,7 @@ func newProjectCommandContext(ctx *command.Context,
 		ParallelApplyEnabled:       parallelApplyEnabled,
 		ParallelPlanEnabled:        parallelPlanEnabled,
 		ParallelPolicyCheckEnabled: parallelPlanEnabled,
+		DependsOn:                  projCfg.DependsOn,
 		AutoplanEnabled:            projCfg.AutoplanEnabled,
 		Steps:                      steps,
 		HeadRepo:                   ctx.HeadRepo,
@@ -297,7 +301,8 @@ func newProjectCommandContext(ctx *command.Context,
 		PolicySets:                 policySets,
 		PolicySetTarget:            ctx.PolicySet,
 		ClearPolicyApproval:        ctx.ClearPolicyApproval,
-		PullReqStatus:              pullStatus,
+		PullReqStatus:              pullReqStatus,
+		PullStatus:                 pullStatus,
 		JobID:                      uuid.New().String(),
 		ExecutionOrderGroup:        projCfg.ExecutionOrderGroup,
 		AbortOnExcecutionOrderFail: abortOnExcecutionOrderFail,

server/events/project_command_runner.go

@@ -607,6 +607,11 @@ func (p *DefaultProjectCommandRunner) doApply(ctx command.ProjectContext) (apply
 		return "", failure, err
 	}
 
+	failure, err = p.CommandRequirementHandler.ValidateProjectDependencies(ctx)
+	if failure != "" || err != nil {
+		return "", failure, err
+	}
+
 	// Acquire internal lock for the directory we're going to operate in.
 	unlockFn, err := p.WorkingDirLocker.TryLock(ctx.Pull.BaseRepo.FullName, ctx.Pull.Num, ctx.Workspace, ctx.RepoRelDir)
 	if err != nil {