feat/next config workflows (#1840)

* supporting workflows in next

backend/controllers/cache.go

@@ -67,7 +67,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 
 	// update the cache here, do it async for immediate response
 	go func() {
-		err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
+		err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
 			diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
 			diggerYmlStr = string(diggerYmlBytes)
 			config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)

backend/controllers/github.go

@@ -553,7 +553,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 	var diggerYmlStr string
 	var dependencyGraph graph.Graph[string, dg_configuration.Project]
 
-	err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
+	err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
 		diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
 		diggerYmlStr = string(diggerYmlBytes)
 		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

backend/utils/github.go

@@ -28,14 +28,18 @@ func createTempDir() string {
 
 type action func(string) error
 
-func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action action) error {
+func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, action action) error {
 	dir := createTempDir()
 	git := NewGitShellWithTokenAuth(dir, token)
 	err := git.Clone(repoUrl, branch)
 	if err != nil {
 		return err
 	}
 
+	if commitHash != "" {
+		git.Checkout(commitHash)
+	}
+
 	defer func() {
 		log.Printf("removing cloned directory %v", dir)
 		ferr := os.RemoveAll(dir)

backend/utils/github_test.go

@@ -14,14 +14,14 @@ func init() {
 
 func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) {
 	f := func(d string) error { return nil }
-	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "invalid-token", f)
+	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", f)
 	assert.NotNil(t, err)
 }
 
 func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) {
 	token := os.Getenv("GITHUB_PAT_TOKEN")
 	f := func(d string) error { return nil }
-	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", token, f)
+	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, f)
 	assert.Nil(t, err)
 }
 
@@ -32,13 +32,13 @@ func TestGithubCloneWithPrivateRepoAndValidTokenThrowsNoError(t *testing.T) {
 		return
 	}
 	f := func(d string) error { return nil }
-	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", token, f)
+	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, f)
 	assert.Nil(t, err)
 }
 
 func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) {
 	token := os.Getenv("GITHUB_PAT_TOKEN")
 	f := func(d string) error { return nil }
-	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", token, f)
+	err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, f)
 	assert.NotNil(t, err)
 }

backend/utils/gitlab.go

@@ -62,7 +62,7 @@ func GetDiggerConfigForBranch(gh GitlabProvider, projectId int, repoFullName str
 		log.Printf("Error getting changed files: %v", err)
 		return "", nil, nil, fmt.Errorf("error getting changed files")
 	}
-	err = CloneGitRepoAndDoAction(cloneUrl, branch, token, func(dir string) error {
+	err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, func(dir string) error {
 		diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
 		diggerYmlStr = string(diggerYmlBytes)
 		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

backend/utils/gitshell.go

@@ -103,6 +103,13 @@ func (g *GitShell) runCommand(args ...string) (string, error) {
 	return strings.TrimSpace(stdout.String()), nil
 }
 
+func (g *GitShell) Checkout(branchOrCommit string) error {
+	args := []string{"checkout"}
+	args = append(args, branchOrCommit)
+	_, err := g.runCommand(args...)
+	return err
+}
+
 // Clone with authentication
 func (g *GitShell) Clone(repoURL, branch string) error {
 	authURL, err := g.formatAuthURL(repoURL)

cli/pkg/spec/spec.go

@@ -43,6 +43,13 @@ func RunSpec(
 		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1)
 	}
 
+	// for additional output reporting
+	diggerOutPath := os.Getenv("DIGGER_OUT")
+	if diggerOutPath == "" {
+		diggerOutPath = os.Getenv("RUNNER_TEMP") + "/digger-out.log"
+		os.Setenv("DIGGER_OUT", diggerOutPath)
+	}
+
 	if spec.Job.Commit != "" {
 		// checking out to the commit ID
 		log.Printf("fetching commit ID %v", spec.Job.Commit)

ee/backend/controllers/gitlab.go

@@ -156,7 +156,7 @@ func handlePushEvent(gh utils.GitlabProvider, payload *gitlab.PushEvent, organis
 		isMainBranch = false
 	}
 
-	err = utils.CloneGitRepoAndDoAction(cloneURL, pushBranch, token, func(dir string) error {
+	err = utils.CloneGitRepoAndDoAction(cloneURL, pushBranch, "", token, func(dir string) error {
 		config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil)
 		if err != nil {
 			log.Printf("ERROR load digger.yml: %v", err)

ee/drift/tasks/github.go

@@ -42,7 +42,7 @@ func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId s
 		return fmt.Errorf("error getting github service")
 	}
 
-	err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
+	err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
 		config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil)
 		if err != nil {
 			log.Printf("ERROR load digger.yml: %v", err)

ee/drift/utils/github.go

@@ -103,7 +103,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 
 	var changedFiles []string = nil
 
-	err = utils2.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
+	err = utils2.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
 		diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
 		diggerYmlStr = string(diggerYmlBytes)
 		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

libs/digger_config/digger_config.go

@@ -30,6 +30,30 @@ type FileSystemTerragruntDirWalker struct {
 type FileSystemModuleDirWalker struct {
 }
 
+func CheckOrCreateDiggerFile(dir string) error {
+	// Check for digger.yml
+	ymlPath := filepath.Join(dir, "digger.yml")
+	yamlPath := filepath.Join(dir, "digger.yaml")
+
+	// Check if either file exists
+	if _, err := os.Stat(ymlPath); err == nil {
+		return nil // digger.yml exists
+	}
+	if _, err := os.Stat(yamlPath); err == nil {
+		return nil // digger.yaml exists
+	}
+
+	// Neither file exists, create digger.yml
+	file, err := os.Create(ymlPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// File is created empty by default
+	return nil
+}
+
 func GetFilesWithExtension(workingDir string, ext string) ([]string, error) {
 	var files []string
 	listOfFiles, err := os.ReadDir(workingDir)
@@ -363,9 +387,6 @@ func LoadDiggerConfigYaml(workingDir string, generateProjects bool, changedFiles
 }
 
 func ValidateDiggerConfigYaml(configYaml *DiggerConfigYaml, fileName string) error {
-	if (configYaml.Projects == nil || len(configYaml.Projects) == 0) && configYaml.GenerateProjectsConfig == nil {
-		return fmt.Errorf("no projects digger_config found in '%s'", fileName)
-	}
 	if configYaml.DependencyConfiguration != nil {
 		if configYaml.DependencyConfiguration.Mode != DependencyConfigurationHard && configYaml.DependencyConfiguration.Mode != DependencyConfigurationSoft {
 			return fmt.Errorf("dependency digger_config mode can only be '%s' or '%s'", DependencyConfigurationHard, DependencyConfigurationSoft)

libs/digger_config/digger_config_test.go

@@ -419,8 +419,9 @@ func TestMissingProjectsReturnsError(t *testing.T) {
 `
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
-	_, _, _, err := LoadDiggerConfig(tempDir, true, nil)
-	assert.ErrorContains(t, err, "no projects digger_config found")
+	config, _, _, err := LoadDiggerConfig(tempDir, true, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, len(config.Projects), 0)
 }
 
 func TestDiggerConfigCustomWorkflow(t *testing.T) {
@@ -941,9 +942,9 @@ func TestDiggerGenerateProjectsEmptyParameters(t *testing.T) {
 	diggerCfg := `
 generate_projects:
 `
-	_, _, _, err := LoadDiggerConfigFromString(diggerCfg, "./")
-	assert.Error(t, err)
-	assert.Equal(t, "no projects digger_config found in 'loaded_yaml_string'", err.Error())
+	config, _, _, err := LoadDiggerConfigFromString(diggerCfg, "./")
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(config.Projects))
 }
 
 // TestDiggerGenerateProjectsTooManyParameters include/exclude and blocks of include/exclude can't be used together

next/controllers/github.go

@@ -80,17 +80,19 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 		err := handlePullRequestEvent(gh, event, d.CiBackendProvider)
 		if err != nil {
 			log.Printf("handlePullRequestEvent error: %v", err)
-			c.String(http.StatusInternalServerError, err.Error())
-			return
 		}
+		c.String(http.StatusAccepted, err.Error())
+		return
+
 	case *github.PushEvent:
 		log.Printf("Got push event for %d", event.Repo.URL)
 		handlePushEventApplyAfterMerge(gh, event)
 		if err != nil {
 			log.Printf("handlePushEvent error: %v", err)
-			c.String(http.StatusInternalServerError, err.Error())
-			return
 		}
+		c.String(http.StatusAccepted, err.Error())
+		return
+
 	default:
 		log.Printf("Unhandled event, event type %v", reflect.TypeOf(event))
 	}
@@ -357,30 +359,23 @@ func handlePullRequestEvent(gh next_utils.GithubClientProvider, payload *github.
 	}
 
 	projectsGraph, err := dg_configuration.CreateProjectDependencyGraph(dgprojects)
+	workflows, err := services.GetWorkflowsForRepoAndBranch(gh, repo.ID, sourceBranch, commitSha)
+	if err != nil {
+		log.Printf("error getting workflows from config: %v", err)
+		return fmt.Errorf("error getting workflows from config")
+	}
 	var config *dg_configuration.DiggerConfig = &dg_configuration.DiggerConfig{
 		ApplyAfterMerge:   true,
 		AllowDraftPRs:     false,
 		CommentRenderMode: "",
 		DependencyConfiguration: dg_configuration.DependencyConfiguration{
 			Mode: dg_configuration.DependencyConfigurationHard,
 		},
-		PrLocks:   false,
-		Projects:  dgprojects,
-		AutoMerge: false,
-		Telemetry: false,
-		Workflows: map[string]dg_configuration.Workflow{
-			"default": dg_configuration.Workflow{
-				EnvVars: nil,
-				Plan:    nil,
-				Apply:   nil,
-				Configuration: &dg_configuration.WorkflowConfiguration{
-					OnPullRequestPushed:           []string{"digger plan"},
-					OnPullRequestClosed:           []string{},
-					OnPullRequestConvertedToDraft: []string{},
-					OnCommitToDefault:             []string{},
-				},
-			},
-		},
+		PrLocks:                    false,
+		Projects:                   dgprojects,
+		AutoMerge:                  false,
+		Telemetry:                  false,
+		Workflows:                  workflows,
 		MentionDriftedProjectsInPR: false,
 		TraverseToNestedProjects:   false,
 	}
@@ -540,7 +535,7 @@ func getDiggerConfigForBranch(gh next_utils.GithubClientProvider, installationId
 		log.Printf("Error getting changed files: %v", err)
 		return "", nil, nil, nil, fmt.Errorf("error getting changed files")
 	}
-	err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
+	err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
 		diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
 		diggerYmlStr = string(diggerYmlBytes)
 		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

next/controllers/github_after_merge.go

@@ -8,6 +8,7 @@ import (
 	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
 	"github.com/diggerhq/digger/libs/scheduler"
 	"github.com/diggerhq/digger/next/dbmodels"
+	"github.com/diggerhq/digger/next/services"
 	nextutils "github.com/diggerhq/digger/next/utils"
 	"github.com/google/go-github/v61/github"
 	"log"
@@ -66,30 +67,23 @@ func handlePushEventApplyAfterMerge(gh nextutils.GithubClientProvider, payload *
 		}
 	}
 	projectsGraph, err := dg_configuration.CreateProjectDependencyGraph(dgprojects)
+	workflows, err := services.GetWorkflowsForRepoAndBranch(gh, repo.ID, targetBranch, commitId)
+	if err != nil {
+		log.Printf("error getting workflows from config: %v", err)
+		return fmt.Errorf("error getting workflows from config")
+	}
 	var config *dg_configuration.DiggerConfig = &dg_configuration.DiggerConfig{
 		ApplyAfterMerge:   true,
 		AllowDraftPRs:     false,
 		CommentRenderMode: "",
 		DependencyConfiguration: dg_configuration.DependencyConfiguration{
 			Mode: dg_configuration.DependencyConfigurationHard,
 		},
-		PrLocks:   false,
-		Projects:  dgprojects,
-		AutoMerge: false,
-		Telemetry: false,
-		Workflows: map[string]dg_configuration.Workflow{
-			"default": dg_configuration.Workflow{
-				EnvVars: nil,
-				Plan:    nil,
-				Apply:   nil,
-				Configuration: &dg_configuration.WorkflowConfiguration{
-					OnPullRequestPushed:           []string{"digger plan"},
-					OnPullRequestClosed:           []string{},
-					OnPullRequestConvertedToDraft: []string{},
-					OnCommitToDefault:             []string{},
-				},
-			},
-		},
+		PrLocks:                    false,
+		Projects:                   dgprojects,
+		AutoMerge:                  false,
+		Telemetry:                  false,
+		Workflows:                  workflows,
 		MentionDriftedProjectsInPR: false,
 		TraverseToNestedProjects:   false,
 	}

next/services/config.go

@@ -0,0 +1,63 @@
+package services
+
+import (
+	"fmt"
+	utils3 "github.com/diggerhq/digger/backend/utils"
+	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
+	"github.com/diggerhq/digger/next/dbmodels"
+	"github.com/diggerhq/digger/next/utils"
+	"log"
+)
+
+func GetWorkflowsForRepoAndBranch(gh utils.GithubClientProvider, repoId int64, branch string, commitHash string) (map[string]dg_configuration.Workflow, error) {
+	r := dbmodels.DB.Query.Repo
+	repo, err := dbmodels.DB.Query.Repo.Where(r.ID.Eq(repoId)).First()
+	if err != nil {
+		log.Printf("could not find repo: %v : %v", repoId, err)
+		return nil, fmt.Errorf("could not find repo: %v: %v", repoId, err)
+	}
+	repoOwner := repo.RepoOrganisation
+	repoFullName := repo.RepoFullName
+	repoName := repo.RepoName
+	orgId := repo.OrganizationID
+
+	appInstallation, err := dbmodels.DB.GetGithubAppInstallationByOrgAndRepo(orgId, repoFullName, dbmodels.GithubAppInstallActive)
+	if err != nil {
+		log.Printf("error retrieving app installation")
+		return nil, fmt.Errorf("error retrieving app installation %v", err)
+	}
+	installationId := appInstallation.GithubInstallationID
+	log.Printf("installation id is: %v", installationId)
+
+	cloneUrl := fmt.Sprintf("https://%v/%v", utils.GetGithubHostname(), repo.RepoFullName)
+
+	_, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
+	if err != nil {
+		log.Printf("could not get github service :%v", err)
+		return nil, fmt.Errorf("could not get github service :%v", err)
+	}
+
+	var config *dg_configuration.DiggerConfig
+
+	err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, commitHash, *token, func(dir string) error {
+		// we create a blank file if it does not exist
+		err := dg_configuration.CheckOrCreateDiggerFile(dir)
+		if err != nil {
+			log.Printf("Error creating blank digger.yml if not exists: %v", err)
+			return err
+		}
+		config, _, _, err = dg_configuration.LoadDiggerConfig(dir, false, nil)
+		if err != nil {
+			log.Printf("Error loading digger config: %v", err)
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("could not load digger config :%v", err)
+		return nil, fmt.Errorf("could not load digger config :%v", err)
+	}
+
+	return config.Workflows, nil
+}