Add github connections to EE (#1803)

* supporting github connections in backend ee

backend/controllers/github.go

@@ -53,7 +53,11 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 	gh := d.GithubClientProvider
 	log.Printf("GithubAppWebHook")
 
-	payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET")))
+	appID := c.GetHeader("X-GitHub-Hook-Installation-Target-ID")
+	log.Printf("app id from header is: %v", appID)
+	_, _, webhookSecret, _, err := d.GithubClientProvider.FetchCredentials(appID)
+
+	payload, err := github.ValidatePayload(c.Request, []byte(webhookSecret))
 	if err != nil {
 		log.Printf("Error validating github app webhook's payload: %v", err)
 		c.String(http.StatusBadRequest, "Error validating github app webhook's payload")
@@ -70,12 +74,17 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 
 	log.Printf("github event type: %v\n", reflect.TypeOf(event))
 
+	appId64, err := strconv.ParseInt(appID, 10, 64)
+	if err != nil {
+		log.Printf("Error converting appId string to int64: %v", err)
+		return
+	}
+
 	switch event := event.(type) {
 	case *github.InstallationEvent:
 		log.Printf("InstallationEvent, action: %v\n", *event.Action)
-
 		if *event.Action == "deleted" {
-			err := handleInstallationDeletedEvent(event)
+			err := handleInstallationDeletedEvent(event, appId64)
 			if err != nil {
 				c.String(http.StatusAccepted, "Failed to handle webhook event.")
 				return
@@ -87,7 +96,7 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 			c.String(http.StatusOK, "OK")
 			return
 		}
-		err := handleIssueCommentEvent(gh, event, d.CiBackendProvider)
+		err = handleIssueCommentEvent(gh, event, d.CiBackendProvider, appId64)
 		if err != nil {
 			log.Printf("handleIssueCommentEvent error: %v", err)
 			c.String(http.StatusAccepted, err.Error())
@@ -105,7 +114,7 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 		}
 	case *github.PullRequestEvent:
 		log.Printf("Got pull request event for %d", *event.PullRequest.ID)
-		err := handlePullRequestEvent(gh, event, d.CiBackendProvider)
+		err = handlePullRequestEvent(gh, event, d.CiBackendProvider, appId64)
 		if err != nil {
 			log.Printf("handlePullRequestEvent error: %v", err)
 			c.String(http.StatusAccepted, err.Error())
@@ -231,11 +240,6 @@ func (d DiggerController) GithubSetupExchangeCode(c *gin.Context) {
 	}
 	log.Printf("Found credentials for GitHub app %v with id %d", *cfg.Name, cfg.GetID())
 
-	_, err = models.DB.CreateGithubApp(cfg.GetName(), cfg.GetID(), cfg.GetHTMLURL())
-	if err != nil {
-		c.Error(fmt.Errorf("Failed to create github app record on callback"))
-	}
-
 	PEM := cfg.GetPEM()
 	PemBase64 := base64.StdEncoding.EncodeToString([]byte(PEM))
 	c.HTML(http.StatusOK, "github_setup.tmpl", gin.H{
@@ -299,10 +303,8 @@ generate_projects:
 	return repo, org, nil
 }
 
-func handleInstallationDeletedEvent(installation *github.InstallationEvent) error {
+func handleInstallationDeletedEvent(installation *github.InstallationEvent, appId int64) error {
 	installationId := *installation.Installation.ID
-	appId := *installation.Installation.AppID
-
 	link, err := models.DB.GetGithubInstallationLinkForInstallationId(installationId)
 	if err != nil {
 		return err
@@ -323,13 +325,7 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent) erro
 	return nil
 }
 
-func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider) error {
-	appId, err := strconv.ParseInt(os.Getenv("GITHUB_APP_ID"), 10, 64)
-	if err != nil {
-		log.Printf("error getting github app installation id: %v", err)
-		return fmt.Errorf("error getting github app installation id")
-	}
-
+func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error {
 	installationId := *payload.Installation.ID
 	repoName := *payload.Repo.Name
 	repoOwner := *payload.Repo.Owner.Login
@@ -645,13 +641,7 @@ func getBatchType(jobs []orchestrator_scheduler.Job) orchestrator_scheduler.Digg
 	}
 }
 
-func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error {
-	appId, err := strconv.ParseInt(os.Getenv("GITHUB_APP_ID"), 10, 64)
-	if err != nil {
-		log.Printf("error getting github app installation id: %v", err)
-		return fmt.Errorf("error getting github app installation id")
-	}
-
+func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error {
 	installationId := *payload.Installation.ID
 	repoName := *payload.Repo.Name
 	repoOwner := *payload.Repo.Owner.Login
@@ -1028,8 +1018,14 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 	installationId := c.Request.URL.Query()["installation_id"][0]
 	//setupAction := c.Request.URL.Query()["setup_action"][0]
 	code := c.Request.URL.Query()["code"][0]
-	clientId := os.Getenv("GITHUB_APP_CLIENT_ID")
-	clientSecret := os.Getenv("GITHUB_APP_CLIENT_SECRET")
+	appId := c.Request.URL.Query().Get("state")
+
+	clientId, clientSecret, _, _, err := d.GithubClientProvider.FetchCredentials(appId)
+	if err != nil {
+		log.Printf("could not fetch credentials for the app: %v", err)
+		c.String(500, "could not find credentials for github app")
+		return
+	}
 
 	installationId64, err := strconv.ParseInt(installationId, 10, 64)
 	if err != nil {

backend/controllers/github_test.go

@@ -589,7 +589,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {
 
 	// migrate tables
 	err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{},
-		&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubApp{}, &models.GithubAppInstallationLink{},
+		&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{},
 		&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.JobToken{})
 	if err != nil {
 		log.Fatal(err)
@@ -706,7 +706,7 @@ func TestGithubHandleIssueCommentEvent(t *testing.T) {
 	var payload github.IssueCommentEvent
 	err := json.Unmarshal([]byte(issueCommentPayload), &payload)
 	assert.NoError(t, err)
-	err = handleIssueCommentEvent(gh, &payload, nil)
+	err = handleIssueCommentEvent(gh, &payload, nil, 0)
 	assert.NoError(t, err)
 
 	jobs, err := models.DB.GetPendingParentDiggerJobs(nil)

backend/go.sum

@@ -299,6 +299,8 @@ github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7l
 github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
 github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
+github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
+github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

backend/migrations/20241107162605.sql

@@ -0,0 +1,2 @@
+-- Modify "github_apps" table
+ALTER TABLE "public"."github_apps" ADD COLUMN "client_id" text NULL, ADD COLUMN "client_secret_encrypted" text NULL, ADD COLUMN "webhook_secret_encrypted" text NULL, ADD COLUMN "private_key_encrypted" text NULL, ADD COLUMN "private_key_base64_encrypted" text NULL, ADD COLUMN "org" text NULL;

backend/migrations/20241107163722.sql

@@ -0,0 +1,21 @@
+-- Create "github_app_connections" table
+CREATE TABLE "public"."github_app_connections" (
+  "id" bigserial NOT NULL,
+  "created_at" timestamptz NULL,
+  "updated_at" timestamptz NULL,
+  "deleted_at" timestamptz NULL,
+  "github_id" bigint NULL,
+  "client_id" text NULL,
+  "client_secret_encrypted" text NULL,
+  "webhook_secret_encrypted" text NULL,
+  "private_key_encrypted" text NULL,
+  "private_key_base64_encrypted" text NULL,
+  "org" text NULL,
+  "name" text NULL,
+  "github_app_url" text NULL,
+  PRIMARY KEY ("id")
+);
+-- Create index "idx_github_app_connections_deleted_at" to table: "github_app_connections"
+CREATE INDEX "idx_github_app_connections_deleted_at" ON "public"."github_app_connections" ("deleted_at");
+-- Drop "github_apps" table
+DROP TABLE "public"."github_apps";

backend/migrations/20241107172343.sql

@@ -0,0 +1,3 @@
+-- Modify "github_app_connections" table
+ALTER TABLE "public"."github_app_connections" ADD COLUMN "organisation_id" bigint NULL, ADD
+ CONSTRAINT "fk_github_app_connections_organisation" FOREIGN KEY ("organisation_id") REFERENCES "public"."organisations" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:A7OaxVcBVM26DpHZ8tH+NLsfDZxrkpsFUxFJbK3Am68=
+h1:LczvJY4QB28QrHCgIm+p3umL5Pkm1NF+iBM0pG6Ij4E=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -30,3 +30,6 @@ h1:A7OaxVcBVM26DpHZ8tH+NLsfDZxrkpsFUxFJbK3Am68=
 20240729155442.sql h1:s7PCALP3SgPz5y9Ya7HkDzYjeN86Q5NniW2YIkOMBrQ=
 20240729155926.sql h1:8vsDrpy/R1UDI+meIp6KoDfhS60t+ngu8aPB+uonFZ4=
 20240729160028.sql h1:snkkxhA2aEQhqBmIhN8l+nPlBhrPOZiPP+dnyhobwD8=
+20241107162605.sql h1:UwkkGz6bQ7nqSCCA9lVS+JS01YDONLVV0yaM8jJqzlQ=
+20241107163722.sql h1:tk28AgXggvpEigTkWMYMxIVDPNdEUFGijaFWBqvlZhA=
+20241107172343.sql h1:wtM1+uJZY6NiiDYabuzj/LAANAV7+xyUCL5U23v3e+c=

backend/models/github.go

@@ -1,12 +1,22 @@
 package models
 
-import "gorm.io/gorm"
+import (
+	"gorm.io/gorm"
+)
 
-type GithubApp struct {
+type GithubAppConnection struct {
 	gorm.Model
-	GithubId     int64
-	Name         string
-	GithubAppUrl string
+	GithubId                  int64 // app id
+	ClientID                  string
+	ClientSecretEncrypted     string
+	WebhookSecretEncrypted    string
+	PrivateKeyEncrypted       string
+	PrivateKeyBase64Encrypted string
+	Org                       string
+	Name                      string
+	GithubAppUrl              string
+	OrganisationID            uint
+	Organisation              Organisation
 }
 
 type GithubAppInstallStatus int

backend/models/scheduler_test.go

@@ -38,7 +38,7 @@ func setupSuiteScheduler(tb testing.TB) (func(tb testing.TB), *Database) {
 
 	// migrate tables
 	err = gdb.AutoMigrate(&Policy{}, &Organisation{}, &Repo{}, &Project{}, &Token{},
-		&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubApp{}, &GithubAppInstallationLink{},
+		&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubAppConnection{}, &GithubAppInstallationLink{},
 		&GithubDiggerJobLink{}, &DiggerJob{}, &DiggerJobParentLink{})
 	if err != nil {
 		log.Fatal(err)

backend/models/storage.go

@@ -436,8 +436,19 @@ func (db *Database) GetGithubAppInstallationLink(installationId int64) (*GithubA
 	return &link, nil
 }
 
-func (db *Database) CreateGithubApp(name string, githubId int64, url string) (*GithubApp, error) {
-	app := GithubApp{Name: name, GithubId: githubId, GithubAppUrl: url}
+func (db *Database) CreateGithubAppConnection(name string, githubId int64, ClientID string, ClientSecretEncrypted string, WebhookSecretEncrypted string, PrivateKeyEncrypted string, PrivateKeyBase64Encrypted string, Org string, url string, orgId uint) (*GithubAppConnection, error) {
+	app := GithubAppConnection{
+		Name:                      name,
+		GithubId:                  githubId,
+		ClientID:                  ClientID,
+		ClientSecretEncrypted:     ClientSecretEncrypted,
+		WebhookSecretEncrypted:    WebhookSecretEncrypted,
+		PrivateKeyEncrypted:       PrivateKeyEncrypted,
+		PrivateKeyBase64Encrypted: PrivateKeyBase64Encrypted,
+		Org:                       Org,
+		GithubAppUrl:              url,
+		OrganisationID:            orgId,
+	}
 	result := db.GormDB.Save(&app)
 	if result.Error != nil {
 		return nil, result.Error
@@ -446,9 +457,19 @@ func (db *Database) CreateGithubApp(name string, githubId int64, url string) (*G
 	return &app, nil
 }
 
+func (db *Database) GetGithubAppConnectionById(id string) (*GithubAppConnection, error) {
+	app := GithubAppConnection{}
+	result := db.GormDB.Where("id = ?", id).Find(&app)
+	if result.Error != nil {
+		log.Printf("Failed to find GitHub App for id: %v, error: %v\n", id, result.Error)
+		return nil, result.Error
+	}
+	return &app, nil
+}
+
 // GetGithubApp return GithubApp by Id
-func (db *Database) GetGithubApp(gitHubAppId any) (*GithubApp, error) {
-	app := GithubApp{}
+func (db *Database) GetGithubAppConnection(gitHubAppId any) (*GithubAppConnection, error) {
+	app := GithubAppConnection{}
 	result := db.GormDB.Where("github_id = ?", gitHubAppId).Find(&app)
 	if result.Error != nil {
 		log.Printf("Failed to find GitHub App for id: %v, error: %v\n", gitHubAppId, result.Error)

backend/models/storage_test.go

@@ -36,7 +36,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *Database, *Organisation) {
 
 	// migrate tables
 	err = gdb.AutoMigrate(&Policy{}, &Organisation{}, &Repo{}, &Project{}, &Token{},
-		&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubApp{}, &GithubAppInstallationLink{},
+		&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubAppConnection{}, &GithubAppInstallationLink{},
 		&GithubDiggerJobLink{}, &DiggerJob{}, &DiggerJobParentLink{})
 	if err != nil {
 		log.Fatal(err)

backend/tasks/runs_test.go

@@ -50,7 +50,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {
 
 	// migrate tables
 	err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{},
-		&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubApp{}, &models.GithubAppInstallationLink{},
+		&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{},
 		&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.DiggerRun{}, &models.DiggerRunQueueItem{})
 	if err != nil {
 		log.Fatal(err)