feat: opt-in gh app integration (#1217)

api/auth/get_token.go

@@ -5,11 +5,13 @@ package auth
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 
 	"github.com/go-vela/server/api/types"
+	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/database"
 	"github.com/go-vela/server/internal/token"
 	"github.com/go-vela/server/scm"
@@ -36,6 +38,14 @@ import (
 //   name: redirect_uri
 //   description: The URL where the user will be sent after authorization
 //   type: string
+// - in: query
+//   name: setup_action
+//   description: The specific setup action callback identifier
+//   type: string
+// - in: query
+//   name: installation_id
+//   description: The specific installation identifier for a GitHub App integration
+//   type: integer
 // responses:
 //   '200':
 //     description: Successfully authenticated
@@ -46,6 +56,10 @@ import (
 //       "$ref": "#/definitions/Token"
 //   '307':
 //     description: Redirected for authentication
+//   '400':
+//     description: Bad Request
+//     schema:
+//       "$ref": "#/definitions/Error"
 //   '401':
 //     description: Unauthorized
 //     schema:
@@ -69,6 +83,32 @@ func GetAuthToken(c *gin.Context) {
 	// capture the OAuth state if present
 	oAuthState := c.Request.FormValue("state")
 
+	// handle scm setup events
+	// setup_action==install represents the GitHub App installation callback redirect
+	if c.Request.FormValue("setup_action") == constants.AppInstallSetupActionInstall {
+		installID, err := strconv.ParseInt(c.Request.FormValue("installation_id"), 10, 0)
+		if err != nil {
+			retErr := fmt.Errorf("unable to parse installation_id: %w", err)
+
+			util.HandleError(c, http.StatusBadRequest, retErr)
+
+			return
+		}
+
+		r, err := scm.FromContext(c).FinishInstallation(ctx, c.Request, installID)
+		if err != nil {
+			retErr := fmt.Errorf("unable to finish installation: %w", err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+
+		c.Redirect(http.StatusTemporaryRedirect, r)
+
+		return
+	}
+
 	// capture the OAuth code if present
 	code := c.Request.FormValue("code")
 	if len(code) == 0 {

api/build/compile_publish.go

@@ -269,6 +269,8 @@ func CompileAndPublish(
 			WithRepo(repo).
 			WithUser(u).
 			WithLabels(cfg.Labels).
+			WithSCM(scm).
+			WithDatabase(database).
 			Compile(ctx, pipelineFile)
 		if err != nil {
 			// format the error message with extra information

api/repo/create.go

@@ -273,6 +273,18 @@ func CreateRepo(c *gin.Context) {
 		}
 	}
 
+	// map this repo to an installation if possible
+	if r.GetInstallID() == 0 {
+		r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r)
+		if err != nil {
+			retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+	}
+
 	// if the repo exists but is inactive
 	if len(dbRepo.GetOrg()) > 0 && !dbRepo.GetActive() {
 		// update the repo owner
@@ -281,6 +293,8 @@ func CreateRepo(c *gin.Context) {
 		dbRepo.SetBranch(r.GetBranch())
 		// activate the repo
 		dbRepo.SetActive(true)
+		// update the install_id
+		dbRepo.SetInstallID(r.GetInstallID())
 
 		// send API call to update the repo
 		// NOTE: not logging modification out separately

api/repo/repair.go

@@ -62,6 +62,8 @@ import (
 
 // RepairRepo represents the API handler to remove
 // and then create a webhook for a repo.
+//
+//nolint:funlen // ignore statement count
 func RepairRepo(c *gin.Context) {
 	// capture middleware values
 	m := c.MustGet("metadata").(*internal.Metadata)
@@ -163,21 +165,48 @@ func RepairRepo(c *gin.Context) {
 		}
 	}
 
+	dirty := false
+
 	// if the repo was previously inactive, mark it as active
 	if !r.GetActive() {
 		r.SetActive(true)
 
-		// send API call to update the repo
+		dirty = true
+
+		l.Tracef("repo %s repaired - set to active", r.GetFullName())
+	}
+
+	// map this repo to an installation, if possible
+	if r.GetInstallID() == 0 {
+		r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r)
+		if err != nil {
+			retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+
+		// install_id was synced
+		if r.GetInstallID() != 0 {
+			dirty = true
+
+			l.Tracef("repo %s repaired - set install_id to %d", r.GetFullName(), r.GetInstallID())
+		}
+	}
+
+	// update the repo in the database, if necessary
+	if dirty {
 		_, err := database.FromContext(c).UpdateRepo(ctx, r)
 		if err != nil {
-			retErr := fmt.Errorf("unable to set repo %s to active: %w", r.GetFullName(), err)
+			retErr := fmt.Errorf("unable to update repo %s during repair: %w", r.GetFullName(), err)
 
 			util.HandleError(c, http.StatusInternalServerError, retErr)
 
 			return
 		}
 
-		l.Infof("repo %s updated - set to active", r.GetFullName())
+		l.Infof("repo %s repaired - database updated", r.GetFullName())
 	}
 
 	c.JSON(http.StatusOK, fmt.Sprintf("repo %s repaired", r.GetFullName()))

api/scm/sync.go

@@ -174,5 +174,31 @@ func SyncRepo(c *gin.Context) {
 		}
 	}
 
+	// map this repo to an installation, if necessary
+	installID := r.GetInstallID()
+
+	r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r)
+	if err != nil {
+		retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err)
+
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+
+		return
+	}
+
+	// install_id was synced
+	if r.GetInstallID() != installID {
+		_, err := database.FromContext(c).UpdateRepo(ctx, r)
+		if err != nil {
+			retErr := fmt.Errorf("unable to update repo %s during repair: %w", r.GetFullName(), err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+
+		l.Tracef("repo %s install_id synced to %d", r.GetFullName(), r.GetInstallID())
+	}
+
 	c.Status(http.StatusNoContent)
 }

api/types/repo.go

@@ -32,6 +32,7 @@ type Repo struct {
 	PipelineType *string   `json:"pipeline_type,omitempty"`
 	PreviousName *string   `json:"previous_name,omitempty"`
 	ApproveBuild *string   `json:"approve_build,omitempty"`
+	InstallID    *int64    `json:"install_id,omitempty"`
 }
 
 // Environment returns a list of environment variables
@@ -345,6 +346,19 @@ func (r *Repo) GetApproveBuild() string {
 	return *r.ApproveBuild
 }
 
+// GetInstallID returns the InstallID field.
+//
+// When the provided Repo type is nil, or the field within
+// the type is nil, it returns the zero value for the field.
+func (r *Repo) GetInstallID() int64 {
+	// return zero value if Repo type or InstallID field is nil
+	if r == nil || r.InstallID == nil {
+		return 0
+	}
+
+	return *r.InstallID
+}
+
 // SetID sets the ID field.
 //
 // When the provided Repo type is nil, it
@@ -618,6 +632,19 @@ func (r *Repo) SetApproveBuild(v string) {
 	r.ApproveBuild = &v
 }
 
+// SetInstallID sets the InstallID field.
+//
+// When the provided Repo type is nil, it
+// will set nothing and immediately return.
+func (r *Repo) SetInstallID(v int64) {
+	// return if Repo type is nil
+	if r == nil {
+		return
+	}
+
+	r.InstallID = &v
+}
+
 // String implements the Stringer interface for the Repo type.
 func (r *Repo) String() string {
 	return fmt.Sprintf(`{
@@ -640,7 +667,8 @@ func (r *Repo) String() string {
   Timeout: %d,
   Topics: %s,
   Trusted: %t,
-  Visibility: %s
+  Visibility: %s,
+  InstallID: %d
 }`,
 		r.GetActive(),
 		r.GetAllowEvents().List(),
@@ -662,5 +690,6 @@ func (r *Repo) String() string {
 		r.GetTopics(),
 		r.GetTrusted(),
 		r.GetVisibility(),
+		r.GetInstallID(),
 	)
 }

api/types/repo_test.go

@@ -303,7 +303,8 @@ func TestTypes_Repo_String(t *testing.T) {
   Timeout: %d,
   Topics: %s,
   Trusted: %t,
-  Visibility: %s
+  Visibility: %s,
+  InstallID: %d
 }`,
 		r.GetActive(),
 		r.GetAllowEvents().List(),
@@ -325,6 +326,7 @@ func TestTypes_Repo_String(t *testing.T) {
 		r.GetTopics(),
 		r.GetTrusted(),
 		r.GetVisibility(),
+		r.GetInstallID(),
 	)
 
 	// run test

api/webhook/post.go

@@ -84,6 +84,7 @@ func PostWebhook(c *gin.Context) {
 	// capture middleware values
 	m := c.MustGet("metadata").(*internal.Metadata)
 	l := c.MustGet("logger").(*logrus.Entry)
+	db := database.FromContext(c)
 	ctx := c.Request.Context()
 
 	l.Debug("webhook received")
@@ -133,6 +134,20 @@ func PostWebhook(c *gin.Context) {
 		return
 	}
 
+	if webhook.Installation != nil {
+		err = scm.FromContext(c).ProcessInstallation(ctx, c.Request, webhook, db)
+		if err != nil {
+			retErr := fmt.Errorf("unable to process installation: %w", err)
+			util.HandleError(c, http.StatusBadRequest, retErr)
+
+			return
+		}
+
+		c.JSON(http.StatusOK, "installation processed successfully")
+
+		return
+	}
+
 	// check if the hook should be skipped
 	if skip, skipReason := webhook.ShouldSkip(); skip {
 		c.JSON(http.StatusOK, fmt.Sprintf("skipping build: %s", skipReason))
@@ -145,8 +160,6 @@ func PostWebhook(c *gin.Context) {
 	l.Debugf("hook generated from SCM: %v", h)
 	l.Debugf("repo generated from SCM: %v", r)
 
-	db := database.FromContext(c)
-
 	// if event is repository event, handle separately and return
 	if strings.EqualFold(h.GetEvent(), constants.EventRepository) {
 		r, err = handleRepositoryEvent(ctx, l, db, m, h, r)

cmd/vela-server/scm.go

@@ -20,16 +20,20 @@ func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) {
 		Address:              c.String("scm.addr"),
 		ClientID:             c.String("scm.client"),
 		ClientSecret:         c.String("scm.secret"),
+		AppID:                c.Int64("scm.app.id"),
+		AppPrivateKey:        c.String("scm.app.private-key"),
+		AppPrivateKeyPath:    c.String("scm.app.private-key.path"),
+		AppPermissions:       c.StringSlice("scm.app.permissions"),
 		ServerAddress:        c.String("server-addr"),
 		ServerWebhookAddress: c.String("scm.webhook.addr"),
 		StatusContext:        c.String("scm.context"),
 		WebUIAddress:         c.String("webui-addr"),
-		Scopes:               c.StringSlice("scm.scopes"),
+		OAuthScopes:          c.StringSlice("scm.scopes"),
 		Tracing:              tc,
 	}
 
 	// setup the scm
 	//
 	// https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#New
-	return scm.New(_setup)
+	return scm.New(c.Context, _setup)
 }

compiler/engine.go

@@ -10,7 +10,9 @@ import (
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
 	"github.com/go-vela/server/compiler/types/yaml"
+	"github.com/go-vela/server/database"
 	"github.com/go-vela/server/internal"
+	"github.com/go-vela/server/scm"
 )
 
 // Engine represents an interface for converting a yaml
@@ -146,6 +148,12 @@ type Engine interface {
 	// WithLabel defines a function that sets
 	// the label(s) in the Engine.
 	WithLabels([]string) Engine
+	// WithSCM defines a function that sets
+	// the scm in the Engine.
+	WithSCM(scm.Service) Engine
+	// WithDatabase defines a function that sets
+	// the database in the Engine.
+	WithDatabase(database.Interface) Engine
 	// WithPrivateGitHub defines a function that sets
 	// the private github client in the Engine.
 	WithPrivateGitHub(context.Context, string, string) Engine

compiler/native/compile.go

@@ -44,6 +44,19 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, *
 		return nil, nil, err
 	}
 
+	// create the netrc using the scm
+	// this has to occur after Parse because the scm configurations might be set in yaml
+	// netrc can be provided directly using WithNetrc for situations like local exec
+	if c.netrc == nil && c.scm != nil {
+		// get the netrc password from the scm
+		netrc, err := c.scm.GetNetrcPassword(ctx, c.db, c.repo, c.user, p.Git)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		c.WithNetrc(netrc)
+	}
+
 	// create the API pipeline object from the yaml configuration
 	_pipeline := p.ToPipelineAPI()
 	_pipeline.SetData(data)