✨ Builder setup page (#189)

* ✨ Resync code repo response 202 * ✨ Sync branches * ✨ Builder setup page
go-sigma · Aug 31, 2023 · dfca34a · dfca34a
1 parent 926b1eb
commit dfca34a
Show file tree

Hide file tree

Showing 29 changed files with 1,517 additions and 43 deletions.
diff --git a/pkg/daemon/coderepo/coderepo.go b/pkg/daemon/coderepo/coderepo.go
@@ -31,6 +31,10 @@ import (
 	"github.com/go-sigma/sigma/pkg/utils"
 )
 
+const (
+	perPage = 100
+)
+
 func init() {
 	utils.PanicIf(daemon.RegisterTask(enums.DaemonCodeRepository, crRunner))
 }

diff --git a/pkg/daemon/coderepo/diff.go b/pkg/daemon/coderepo/diff.go
@@ -25,7 +25,7 @@ import (
 	"github.com/go-sigma/sigma/pkg/dal/query"
 )
 
-func (cr codeRepository) diff(ctx context.Context, user3rdPartyObj *models.User3rdParty, newRepos []*models.CodeRepository) error {
+func (cr codeRepository) diff(ctx context.Context, user3rdPartyObj *models.User3rdParty, newRepos []*models.CodeRepository, branchMap map[string][]*models.CodeRepositoryBranch) error {
 	codeRepositoryService := cr.codeRepositoryServiceFactory.New()
 	oldRepos, err := codeRepositoryService.ListAll(ctx, user3rdPartyObj.ID)
 	if err != nil {
@@ -170,5 +170,81 @@ func (cr codeRepository) diff(ctx context.Context, user3rdPartyObj *models.User3
 	if err != nil {
 		return err
 	}
-	return nil
+	return cr.diffBranch(ctx, user3rdPartyObj, branchMap)
+}
+
+func (cr codeRepository) diffBranch(ctx context.Context, user3rdPartyObj *models.User3rdParty, branchMap map[string][]*models.CodeRepositoryBranch) error {
+	if len(branchMap) == 0 {
+		return nil
+	}
+	codeRepositoryService := cr.codeRepositoryServiceFactory.New()
+	repositoryObjs, err := codeRepositoryService.ListAll(ctx, user3rdPartyObj.ID)
+	if err != nil {
+		log.Error().Err(err).Msg("List all repositories failed")
+		return fmt.Errorf("List all repositories failed: %v", err)
+	}
+
+	var needInsertBranches []*models.CodeRepositoryBranch
+	var needDelBranches []int64
+	for _, repo := range repositoryObjs {
+		oldBranches, _, err := codeRepositoryService.ListBranchesWithoutPagination(ctx, repo.ID)
+		if err != nil {
+			log.Error().Err(err).Int64("id", repo.ID).Msg("List repo branches failed")
+			return fmt.Errorf("List repo branches failed: %v", err)
+		}
+		if len(branchMap[repo.RepositoryID]) == 0 {
+			var bs []*models.CodeRepositoryBranch
+			for _, b := range branchMap[repo.RepositoryID] {
+				bs = append(bs, &models.CodeRepositoryBranch{CodeRepositoryID: repo.ID, Name: b.Name})
+			}
+			needInsertBranches = append(needInsertBranches, bs...)
+			continue
+		}
+
+		for _, oldB := range oldBranches {
+			found := false
+			for _, newB := range branchMap[repo.RepositoryID] {
+				if newB.Name == oldB.Name {
+					found = true
+				}
+			}
+			if !found {
+				needDelBranches = append(needDelBranches, oldB.ID)
+			}
+		}
+
+		for _, newB := range branchMap[repo.RepositoryID] {
+			found := false
+			for _, oldB := range oldBranches {
+				if newB.Name == oldB.Name {
+					found = true
+				}
+			}
+			if !found {
+				needInsertBranches = append(needInsertBranches, &models.CodeRepositoryBranch{
+					CodeRepositoryID: repo.ID,
+					Name:             newB.Name,
+				})
+			}
+		}
+	}
+	err = query.Q.Transaction(func(tx *query.Query) error {
+		codeRepositoryService := cr.codeRepositoryServiceFactory.New(tx)
+		if len(needInsertBranches) > 0 {
+			err := codeRepositoryService.CreateBranchesInBatches(ctx, needInsertBranches)
+			if err != nil {
+				log.Error().Err(err).Msg("Create new branches failed")
+				return fmt.Errorf("Create new branches failed: %v", err)
+			}
+		}
+		if len(needDelBranches) > 0 {
+			err := codeRepositoryService.DeleteBranchesInBatches(ctx, needDelBranches)
+			if err != nil {
+				log.Error().Err(err).Msg("Delete branches failed")
+				return fmt.Errorf("Delete branches failed: %v", err)
+			}
+		}
+		return nil
+	})
+	return err
 }
diff --git a/pkg/daemon/coderepo/gitea.go b/pkg/daemon/coderepo/gitea.go
@@ -92,5 +92,5 @@ func (cr codeRepository) gitea(ctx context.Context, user3rdPartyObj *models.User
 		})
 	}
 
-	return cr.diff(ctx, user3rdPartyObj, newRepos)
+	return cr.diff(ctx, user3rdPartyObj, newRepos, nil)
 }
diff --git a/pkg/daemon/coderepo/github.go b/pkg/daemon/coderepo/github.go
@@ -22,15 +22,12 @@ import (
 
 	"github.com/google/go-github/v53/github"
 	"github.com/rs/zerolog/log"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/go-sigma/sigma/pkg/dal/models"
 	"github.com/go-sigma/sigma/pkg/utils/ptr"
 )
 
-const (
-	perPage = 100
-)
-
 func (cr codeRepository) github(ctx context.Context, user3rdPartyObj *models.User3rdParty) error {
 	client := github.NewTokenClient(ctx, ptr.To(user3rdPartyObj.Token))
 
@@ -110,5 +107,44 @@ func (cr codeRepository) github(ctx context.Context, user3rdPartyObj *models.Use
 		newRepos = append(newRepos, repo)
 	}
 
-	return cr.diff(ctx, user3rdPartyObj, newRepos)
+	var blockedRepo = sets.NewString()
+
+	var branchMap = make(map[string][]*models.CodeRepositoryBranch)
+	for _, r := range newRepos {
+		var branches []*models.CodeRepositoryBranch
+		page = 1
+		for {
+			bs, _, err := client.Repositories.ListBranches(ctx, r.Owner, r.Name, &github.BranchListOptions{ListOptions: github.ListOptions{Page: page, PerPage: perPage}})
+			if err != nil {
+				if strings.Contains(err.Error(), "Repository access blocked") {
+					blockedRepo.Insert(r.RepositoryID)
+				} else {
+					log.Error().Err(err).Str("owner", r.Owner).Str("repo", r.Name).Msg("List branches failed")
+					return fmt.Errorf("List branches for repo(%s/%s) failed: %v", r.Owner, r.Name, err)
+				}
+			}
+			var bsObj = make([]*models.CodeRepositoryBranch, 0, len(bs))
+			for _, b := range bs {
+				bsObj = append(bsObj, &models.CodeRepositoryBranch{
+					Name: b.GetName(),
+				})
+			}
+			branches = append(branches, bsObj...)
+			if len(bs) < perPage {
+				break
+			}
+			page++
+		}
+		branchMap[r.RepositoryID] = branches
+	}
+
+	repoFiltered := make([]*models.CodeRepository, 0, len(newRepos))
+	for _, r := range newRepos {
+		if blockedRepo.Has(r.RepositoryID) {
+			continue
+		}
+		repoFiltered = append(repoFiltered, r)
+	}
+
+	return cr.diff(ctx, user3rdPartyObj, repoFiltered, branchMap)
 }
diff --git a/pkg/daemon/coderepo/gitlab.go b/pkg/daemon/coderepo/gitlab.go
@@ -113,5 +113,30 @@ func (cr codeRepository) gitlab(ctx context.Context, user3rdPartyObj *models.Use
 		newRepos = append(newRepos, repo)
 	}
 
-	return cr.diff(ctx, user3rdPartyObj, newRepos)
+	var branchMap = make(map[string][]*models.CodeRepositoryBranch)
+	for _, r := range newRepos {
+		var branches []*models.CodeRepositoryBranch
+		page = 1
+		for {
+			bs, _, err := client.Branches.ListBranches(r.RepositoryID, &gitlab.ListBranchesOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}})
+			if err != nil {
+				log.Error().Err(err).Str("owner", r.Owner).Str("repo", r.Name).Msg("List branches failed")
+				return fmt.Errorf("List branches for repo(%s/%s) failed: %v", r.Owner, r.Name, err)
+			}
+			var bsObj = make([]*models.CodeRepositoryBranch, 0, len(bs))
+			for _, b := range bs {
+				bsObj = append(bsObj, &models.CodeRepositoryBranch{
+					Name: b.Name,
+				})
+			}
+			branches = append(branches, bsObj...)
+			if len(bs) < perPage {
+				break
+			}
+			page++
+		}
+		branchMap[r.RepositoryID] = branches
+	}
+
+	return cr.diff(ctx, user3rdPartyObj, newRepos, branchMap)
 }
diff --git a/pkg/dal/cmd/gen.go b/pkg/dal/cmd/gen.go
@@ -32,6 +32,7 @@ func main() {
 		models.User3rdParty{},
 		models.UserRecoverCode{},
 		models.CodeRepository{},
+		models.CodeRepositoryBranch{},
 		models.CodeRepositoryOwner{},
 		models.CodeRepositoryCloneCredential{},
 		models.Audit{},

diff --git a/pkg/dal/dao/code_repository.go b/pkg/dal/dao/code_repository.go
@@ -36,6 +36,8 @@ type CodeRepositoryService interface {
 	CreateInBatches(ctx context.Context, codeRepositories []*models.CodeRepository) error
 	// CreateOwnersInBatches creates new code repository owner records in the database
 	CreateOwnersInBatches(ctx context.Context, codeRepositoryOwners []*models.CodeRepositoryOwner) error
+	// CreateBranchesInBatches ...
+	CreateBranchesInBatches(ctx context.Context, branches []*models.CodeRepositoryBranch) error
 	// UpdateInBatches updates code repository records in the database
 	UpdateInBatches(ctx context.Context, codeRepositories []*models.CodeRepository) error
 	// UpdateOwnersInBatches updates code repository owner records in the database
@@ -44,6 +46,8 @@ type CodeRepositoryService interface {
 	DeleteInBatches(ctx context.Context, ids []int64) error
 	// DeleteOwnerInBatches deletes code repository owner records in the database
 	DeleteOwnerInBatches(ctx context.Context, ids []int64) error
+	// DeleteBranchesInBatches ...
+	DeleteBranchesInBatches(ctx context.Context, ids []int64) error
 	// ListAll lists all code repository records in the database
 	ListAll(ctx context.Context, user3rdPartyID int64) ([]*models.CodeRepository, error)
 	// ListOwnersAll lists all code repository owners records in the database
@@ -52,6 +56,8 @@ type CodeRepositoryService interface {
 	ListWithPagination(ctx context.Context, userID int64, provider enums.Provider, owner, name *string, pagination types.Pagination, sort types.Sortable) ([]*models.CodeRepository, int64, error)
 	// ListOwnerWithoutPagination list code repositories without pagination
 	ListOwnerWithoutPagination(ctx context.Context, userID int64, provider enums.Provider, owner *string) ([]*models.CodeRepositoryOwner, int64, error)
+	// ListBranchesWithoutPagination ...
+	ListBranchesWithoutPagination(ctx context.Context, codeRepositoryID int64) ([]*models.CodeRepositoryBranch, int64, error)
 }
 
 type codeRepositoryService struct {
@@ -90,6 +96,11 @@ func (s *codeRepositoryService) CreateOwnersInBatches(ctx context.Context, codeR
 	return s.tx.CodeRepositoryOwner.WithContext(ctx).CreateInBatches(codeRepositoryOwners, consts.InsertBatchSize)
 }
 
+// CreateBranchesInBatches ...
+func (s *codeRepositoryService) CreateBranchesInBatches(ctx context.Context, branches []*models.CodeRepositoryBranch) error {
+	return s.tx.CodeRepositoryBranch.WithContext(ctx).CreateInBatches(branches, consts.InsertBatchSize)
+}
+
 // UpdateInBatches updates code repository records in the database
 func (s *codeRepositoryService) UpdateInBatches(ctx context.Context, codeRepositories []*models.CodeRepository) error {
 	for _, cr := range codeRepositories {
@@ -148,6 +159,18 @@ func (s *codeRepositoryService) DeleteOwnerInBatches(ctx context.Context, ids []
 	return nil
 }
 
+// DeleteBranchesInBatches ...
+func (s *codeRepositoryService) DeleteBranchesInBatches(ctx context.Context, ids []int64) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	_, err := s.tx.CodeRepositoryBranch.WithContext(ctx).Where(s.tx.CodeRepositoryBranch.ID.In(ids...)).Delete()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // ListAll lists all code repository records in the database
 func (s *codeRepositoryService) ListAll(ctx context.Context, user3rdPartyID int64) ([]*models.CodeRepository, error) {
 	return s.tx.CodeRepository.WithContext(ctx).Where(s.tx.CodeRepository.User3rdPartyID.Eq(user3rdPartyID)).Find()
@@ -203,3 +226,8 @@ func (s *codeRepositoryService) ListOwnerWithoutPagination(ctx context.Context,
 
 	return query.FindByPage(-1, -1)
 }
+
+// ListBranchesWithoutPagination ...
+func (s *codeRepositoryService) ListBranchesWithoutPagination(ctx context.Context, codeRepositoryID int64) ([]*models.CodeRepositoryBranch, int64, error) {
+	return s.tx.CodeRepositoryBranch.WithContext(ctx).Where(s.tx.CodeRepositoryBranch.CodeRepositoryID.Eq(codeRepositoryID)).FindByPage(-1, -1)
+}
diff --git a/pkg/dal/dao/mocks/code_repository.go b/pkg/dal/dao/mocks/code_repository.go
diff --git a/pkg/dal/migrations/mysql/0001_initialize.up.sql b/pkg/dal/migrations/mysql/0001_initialize.up.sql
@@ -71,6 +71,17 @@ CREATE TABLE IF NOT EXISTS `code_repositories` (
   CONSTRAINT `code_repositories_unique_with_name` UNIQUE (`user_3rdparty_id`, `owner_id`, `repository_id`, `deleted_at`)
 );
 
+CREATE TABLE IF NOT EXISTS `code_repository_branches` (
+  `id` bigint AUTO_INCREMENT PRIMARY KEY,
+  `code_repository_id` bigint NOT NULL,
+  `name` varchar(256) NOT NULL,
+  `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `deleted_at` bigint NOT NULL DEFAULT 0,
+  FOREIGN KEY (`code_repository_id`) REFERENCES `code_repositories` (`id`),
+  CONSTRAINT `code_repository_branches_unique_with_name` UNIQUE (`code_repository_id`, `name`, `deleted_at`)
+);
+
 CREATE TABLE IF NOT EXISTS `user_recover_codes` (
   `id` bigint AUTO_INCREMENT PRIMARY KEY,
   `user_id` bigint NOT NULL,

diff --git a/pkg/dal/migrations/postgresql/0001_initialize.up.sql b/pkg/dal/migrations/postgresql/0001_initialize.up.sql
@@ -91,6 +91,17 @@ CREATE TABLE IF NOT EXISTS "code_repositories" (
   CONSTRAINT "code_repositories_unique_with_name" UNIQUE ("user_3rdparty_id", "owner_id", "repository_id", "deleted_at")
 );
 
+CREATE TABLE IF NOT EXISTS "code_repository_branches" (
+  "id" bigserial PRIMARY KEY,
+  "code_repository_id" integer NOT NULL,
+  "name" varchar(256) NOT NULL,
+  "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  "deleted_at" integer NOT NULL DEFAULT 0,
+  FOREIGN KEY ("code_repository_id") REFERENCES "code_repositories" ("id"),
+  CONSTRAINT "code_repository_branches_unique_with_name" UNIQUE ("code_repository_id", "name", "deleted_at")
+);
+
 CREATE TABLE IF NOT EXISTS "user_recover_codes" (
   "id" bigserial PRIMARY KEY,
   "user_id" bigint NOT NULL,