Skip to content

Commit

Permalink
Add new API endpoints for push mirrors management (go-gitea#19841)
Browse files Browse the repository at this point in the history
- Add a new push mirror to specific repository
- Sync now ( send all the changes to the configured push mirrors )
- Get list of all push mirrors of a repository
- Get a push mirror by ID
- Delete push mirror by ID

Signed-off-by: Mohamed Sekour <[email protected]>
Signed-off-by: Andrew Thornton <[email protected]>
Co-authored-by: zeripath <[email protected]>
  • Loading branch information
2 people authored and Sysoev, Vladimir committed Aug 10, 2022
1 parent d6f15bf commit 524bfc4
Show file tree
Hide file tree
Showing 14 changed files with 787 additions and 44 deletions.
5 changes: 3 additions & 2 deletions integrations/mirror_push_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"testing"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
Expand Down Expand Up @@ -47,7 +48,7 @@ func testMirrorPush(t *testing.T, u *url.URL) {

doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t)

mirrors, err := repo_model.GetPushMirrorsByRepoID(srcRepo.ID)
mirrors, _, err := repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
assert.NoError(t, err)
assert.Len(t, mirrors, 1)

Expand All @@ -72,7 +73,7 @@ func testMirrorPush(t *testing.T, u *url.URL) {

// Cleanup
doRemovePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword, int(mirrors[0].ID))(t)
mirrors, err = repo_model.GetPushMirrorsByRepoID(srcRepo.ID)
mirrors, _, err = repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
assert.NoError(t, err)
assert.Len(t, mirrors, 0)
}
Expand Down
73 changes: 49 additions & 24 deletions models/repo/pushmirror.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@
package repo

import (
"context"
"errors"
"time"

"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"

"xorm.io/builder"
)

// ErrPushMirrorNotExist mirror does not exist error
Expand All @@ -29,6 +32,25 @@ type PushMirror struct {
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
LastError string `xorm:"text"`
}
type PushMirrorOptions struct {
ID int64
RepoID int64
RemoteName string
}

func (opts *PushMirrorOptions) toConds() builder.Cond {
cond := builder.NewCond()
if opts.RepoID > 0 {
cond = cond.And(builder.Eq{"repo_id": opts.RepoID})
}
if opts.RemoteName != "" {
cond = cond.And(builder.Eq{"remote_name": opts.RemoteName})
}
if opts.ID > 0 {
cond = cond.And(builder.Eq{"id": opts.ID})
}
return cond
}

func init() {
db.RegisterModel(new(PushMirror))
Expand All @@ -53,45 +75,48 @@ func (m *PushMirror) GetRemoteName() string {
}

// InsertPushMirror inserts a push-mirror to database
func InsertPushMirror(m *PushMirror) error {
_, err := db.GetEngine(db.DefaultContext).Insert(m)
func InsertPushMirror(ctx context.Context, m *PushMirror) error {
_, err := db.GetEngine(ctx).Insert(m)
return err
}

// UpdatePushMirror updates the push-mirror
func UpdatePushMirror(m *PushMirror) error {
_, err := db.GetEngine(db.DefaultContext).ID(m.ID).AllCols().Update(m)
return err
}

// DeletePushMirrorByID deletes a push-mirrors by ID
func DeletePushMirrorByID(ID int64) error {
_, err := db.GetEngine(db.DefaultContext).ID(ID).Delete(&PushMirror{})
func UpdatePushMirror(ctx context.Context, m *PushMirror) error {
_, err := db.GetEngine(ctx).ID(m.ID).AllCols().Update(m)
return err
}

// DeletePushMirrorsByRepoID deletes all push-mirrors by repoID
func DeletePushMirrorsByRepoID(repoID int64) error {
_, err := db.GetEngine(db.DefaultContext).Delete(&PushMirror{RepoID: repoID})
return err
func DeletePushMirrors(ctx context.Context, opts PushMirrorOptions) error {
if opts.RepoID > 0 {
_, err := db.GetEngine(ctx).Where(opts.toConds()).Delete(&PushMirror{})
return err
}
return errors.New("repoID required and must be set")
}

// GetPushMirrorByID returns push-mirror information.
func GetPushMirrorByID(ID int64) (*PushMirror, error) {
m := &PushMirror{}
has, err := db.GetEngine(db.DefaultContext).ID(ID).Get(m)
func GetPushMirror(ctx context.Context, opts PushMirrorOptions) (*PushMirror, error) {
mirror := &PushMirror{}
exist, err := db.GetEngine(ctx).Where(opts.toConds()).Get(mirror)
if err != nil {
return nil, err
} else if !has {
} else if !exist {
return nil, ErrPushMirrorNotExist
}
return m, nil
return mirror, nil
}

// GetPushMirrorsByRepoID returns push-mirror information of a repository.
func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
func GetPushMirrorsByRepoID(ctx context.Context, repoID int64, listOptions db.ListOptions) ([]*PushMirror, int64, error) {
sess := db.GetEngine(ctx).Where("repo_id = ?", repoID)
if listOptions.Page != 0 {
sess = db.SetSessionPagination(sess, &listOptions)
mirrors := make([]*PushMirror, 0, listOptions.PageSize)
count, err := sess.FindAndCount(&mirrors)
return mirrors, count, err
}
mirrors := make([]*PushMirror, 0, 10)
return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors)
count, err := sess.FindAndCount(&mirrors)
return mirrors, count, err
}

// GetPushMirrorsSyncedOnCommit returns push-mirrors for this repo that should be updated by new commits
Expand All @@ -103,8 +128,8 @@ func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) {
}

// PushMirrorsIterate iterates all push-mirror repositories.
func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
func PushMirrorsIterate(ctx context.Context, limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(ctx).
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
And("`interval` != 0").
OrderBy("last_update ASC").
Expand Down
9 changes: 5 additions & 4 deletions models/repo/pushmirror_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"testing"
"time"

"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/timeutil"
Expand All @@ -20,28 +21,28 @@ func TestPushMirrorsIterate(t *testing.T) {

now := timeutil.TimeStampNow()

repo_model.InsertPushMirror(&repo_model.PushMirror{
repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{
RemoteName: "test-1",
LastUpdateUnix: now,
Interval: 1,
})

long, _ := time.ParseDuration("24h")
repo_model.InsertPushMirror(&repo_model.PushMirror{
repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{
RemoteName: "test-2",
LastUpdateUnix: now,
Interval: long,
})

repo_model.InsertPushMirror(&repo_model.PushMirror{
repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{
RemoteName: "test-3",
LastUpdateUnix: now,
Interval: 0,
})

time.Sleep(1 * time.Millisecond)

repo_model.PushMirrorsIterate(1, func(idx int, bean interface{}) error {
repo_model.PushMirrorsIterate(db.DefaultContext, 1, func(idx int, bean interface{}) error {
m, ok := bean.(*repo_model.PushMirror)
assert.True(t, ok)
assert.Equal(t, "test-1", m.RemoteName)
Expand Down
2 changes: 1 addition & 1 deletion modules/context/repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ func repoAssignment(ctx *Context, repo *repo_model.Repository) {
}
}

pushMirrors, err := repo_model.GetPushMirrorsByRepoID(repo.ID)
pushMirrors, _, err := repo_model.GetPushMirrorsByRepoID(ctx, repo.ID, db.ListOptions{})
if err != nil {
ctx.ServerError("GetPushMirrorsByRepoID", err)
return
Expand Down
39 changes: 39 additions & 0 deletions modules/convert/mirror.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package convert

import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
api "code.gitea.io/gitea/modules/structs"
)

// ToPushMirror convert from repo_model.PushMirror and remoteAddress to api.TopicResponse
func ToPushMirror(pm *repo_model.PushMirror) (*api.PushMirror, error) {
repo := pm.GetRepository()
remoteAddress, err := getRemoteAddress(repo, pm.RemoteName)
if err != nil {
return nil, err
}
return &api.PushMirror{
RepoName: repo.Name,
RemoteName: pm.RemoteName,
RemoteAddress: remoteAddress,
CreatedUnix: pm.CreatedUnix.FormatLong(),
LastUpdateUnix: pm.LastUpdateUnix.FormatLong(),
LastError: pm.LastError,
Interval: pm.Interval.String(),
}, nil
}

func getRemoteAddress(repo *repo_model.Repository, remoteName string) (string, error) {
url, err := git.GetRemoteURL(git.DefaultContext, repo.RepoPath(), remoteName)
if err != nil {
return "", err
}
// remove confidential information
url.User = nil
return url.String(), nil
}
25 changes: 25 additions & 0 deletions modules/structs/mirror.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package structs

// CreatePushMirrorOption represents need information to create a push mirror of a repository.
type CreatePushMirrorOption struct {
RemoteAddress string `json:"remote_address"`
RemoteUsername string `json:"remote_username"`
RemotePassword string `json:"remote_password"`
Interval string `json:"interval"`
}

// PushMirror represents information of a push mirror
// swagger:model
type PushMirror struct {
RepoName string `json:"repo_name"`
RemoteName string `json:"remote_name"`
RemoteAddress string `json:"remote_address"`
CreatedUnix string `json:"created"`
LastUpdateUnix string `json:"last_update"`
LastError string `json:"last_error"`
Interval string `json:"interval"`
}
9 changes: 9 additions & 0 deletions routers/api/v1/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -982,6 +982,15 @@ func Routes() *web.Route {
})
}, reqRepoReader(unit.TypeReleases))
m.Post("/mirror-sync", reqToken(), reqRepoWriter(unit.TypeCode), repo.MirrorSync)
m.Post("/push_mirrors-sync", reqAdmin(), repo.PushMirrorSync)
m.Group("/push_mirrors", func() {
m.Combo("").Get(repo.ListPushMirrors).
Post(bind(api.CreatePushMirrorOption{}), repo.AddPushMirror)
m.Combo("/{name}").
Delete(repo.DeletePushMirrorByRemoteName).
Get(repo.GetPushMirrorByName)
}, reqAdmin())

m.Get("/editorconfig/{filename}", context.ReferencesGitRepo(), context.RepoRefForAPI, reqRepoReader(unit.TypeCode), repo.GetEditorconfig)
m.Group("/pulls", func() {
m.Combo("").Get(repo.ListPullRequests).
Expand Down
Loading

0 comments on commit 524bfc4

Please sign in to comment.