From d6538112f4974ac281de4f8e1c48079b1b1b6cc7 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 17 May 2019 17:29:13 +0100 Subject: [PATCH 01/25] Move hook functionality internally --- cmd/hook.go | 123 ++++++---------------- models/helper_environment.go | 2 +- modules/private/hook.go | 72 +++++++++++++ routers/private/hook.go | 193 +++++++++++++++++++++++++++++++++++ routers/private/internal.go | 2 + 5 files changed, 302 insertions(+), 90 deletions(-) create mode 100644 modules/private/hook.go create mode 100644 routers/private/hook.go diff --git a/cmd/hook.go b/cmd/hook.go index f8bd34c4e995..5b48e20f5bf8 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -8,15 +8,14 @@ import ( "bufio" "bytes" "fmt" + "net/http" "os" "strconv" "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" - "code.gitea.io/gitea/modules/util" "github.com/urfave/cli" ) @@ -62,12 +61,10 @@ func runHookPreReceive(c *cli.Context) error { setup("hooks/pre-receive.log") // the environment setted on serv command - repoID, _ := strconv.ParseInt(os.Getenv(models.ProtectedBranchRepoID), 10, 64) isWiki := (os.Getenv(models.EnvRepoIsWiki) == "true") username := os.Getenv(models.EnvRepoUsername) reponame := os.Getenv(models.EnvRepoName) - userIDStr := os.Getenv(models.EnvPusherID) - repoPath := models.RepoPath(username, reponame) + userID, _ := strconv.ParseInt(os.Getenv(models.EnvPusherID), 10, 64) buf := bytes.NewBuffer(nil) scanner := bufio.NewScanner(os.Stdin) @@ -91,35 +88,17 @@ func runHookPreReceive(c *cli.Context) error { // If the ref is a branch, check if it's protected if strings.HasPrefix(refFullName, git.BranchPrefix) { - branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) - protectBranch, err := private.GetProtectedBranchBy(repoID, branchName) - if err != nil { - fail("Internal error", fmt.Sprintf("retrieve protected branches information failed: %v", err)) - } - - if protectBranch != nil && protectBranch.IsProtected() { - // check and deletion - if newCommitID == git.EmptySHA { - fail(fmt.Sprintf("branch %s is protected from deletion", branchName), "") - } - - // detect force push - if git.EmptySHA != oldCommitID { - output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDir(repoPath) - if err != nil { - fail("Internal error", "Fail to detect force push: %v", err) - } else if len(output) > 0 { - fail(fmt.Sprintf("branch %s is protected from force push", branchName), "") - } - } - - userID, _ := strconv.ParseInt(userIDStr, 10, 64) - canPush, err := private.CanUserPush(protectBranch.ID, userID) - if err != nil { - fail("Internal error", "Fail to detect user can push: %v", err) - } else if !canPush { - fail(fmt.Sprintf("protected branch %s can not be pushed to", branchName), "") - } + statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{ + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + RefFullName: refFullName, + UserID: userID, + }) + switch statusCode { + case http.StatusInternalServerError: + fail("Internal Server Error", msg) + case http.StatusForbidden: + fail(msg, "") } } } @@ -145,7 +124,6 @@ func runHookPostReceive(c *cli.Context) error { setup("hooks/post-receive.log") // the environment setted on serv command - repoID, _ := strconv.ParseInt(os.Getenv(models.ProtectedBranchRepoID), 10, 64) repoUser := os.Getenv(models.EnvRepoUsername) isWiki := (os.Getenv(models.EnvRepoIsWiki) == "true") repoName := os.Getenv(models.EnvRepoName) @@ -172,64 +150,31 @@ func runHookPostReceive(c *cli.Context) error { newCommitID := string(fields[1]) refFullName := string(fields[2]) - // Only trigger activity updates for changes to branches or - // tags. Updates to other refs (eg, refs/notes, refs/changes, - // or other less-standard refs spaces are ignored since there - // may be a very large number of them). - if strings.HasPrefix(refFullName, git.BranchPrefix) || strings.HasPrefix(refFullName, git.TagPrefix) { - if err := private.PushUpdate(models.PushUpdateOptions{ - RefFullName: refFullName, - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - PusherID: pusherID, - PusherName: pusherName, - RepoUserName: repoUser, - RepoName: repoName, - }); err != nil { - log.GitLogger.Error("Update: %v", err) - } - } - - if newCommitID != git.EmptySHA && strings.HasPrefix(refFullName, git.BranchPrefix) { - branch := strings.TrimPrefix(refFullName, git.BranchPrefix) - repo, pullRequestAllowed, err := private.GetRepository(repoID) - if err != nil { - log.GitLogger.Error("get repo: %v", err) - break - } - if !pullRequestAllowed { - break - } - - baseRepo := repo - if repo.IsFork { - baseRepo = repo.BaseRepo - } - - if !repo.IsFork && branch == baseRepo.DefaultBranch { - break - } + res, err := private.HookPostReceive(repoUser, repoName, private.HookOptions{ + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + RefFullName: refFullName, + UserID: pusherID, + UserName: pusherName, + }) - pr, err := private.ActivePullRequest(baseRepo.ID, repo.ID, baseRepo.DefaultBranch, branch) - if err != nil { - log.GitLogger.Error("get active pr: %v", err) - break - } + if res == nil { + fail("Internal Server Error", err) + } - fmt.Fprintln(os.Stderr, "") - if pr == nil { - if repo.IsFork { - branch = fmt.Sprintf("%s:%s", repo.OwnerName, branch) - } - fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", branch) - fmt.Fprintf(os.Stderr, " %s/compare/%s...%s\n", baseRepo.HTMLURL(), util.PathEscapeSegments(baseRepo.DefaultBranch), util.PathEscapeSegments(branch)) - } else { - fmt.Fprint(os.Stderr, "Visit the existing pull request:\n") - fmt.Fprintf(os.Stderr, " %s/pulls/%d\n", baseRepo.HTMLURL(), pr.Index) - } - fmt.Fprintln(os.Stderr, "") + if res["message"] == false { + continue } + fmt.Fprintln(os.Stderr, "") + if res["create"] == true { + fmt.Fprintf(os.Stderr, "Create a new pull request for '%s':\n", res["branch"]) + fmt.Fprintf(os.Stderr, " %s\n", res["url"]) + } else { + fmt.Fprint(os.Stderr, "Visit the existing pull request:\n") + fmt.Fprintf(os.Stderr, " %s\n", res["url"]) + } + fmt.Fprintln(os.Stderr, "") } return nil diff --git a/models/helper_environment.go b/models/helper_environment.go index 737a9a68c3a3..199eb6062d52 100644 --- a/models/helper_environment.go +++ b/models/helper_environment.go @@ -27,7 +27,7 @@ func PushingEnvironment(doer *User, repo *Repository) []string { "GIT_COMMITTER_NAME="+sig.Name, "GIT_COMMITTER_EMAIL="+sig.Email, EnvRepoName+"="+repo.Name, - EnvRepoUsername+"="+repo.OwnerName, + EnvRepoUsername+"="+repo.MustOwnerName(), EnvRepoIsWiki+"="+isWiki, EnvPusherName+"="+doer.Name, EnvPusherID+"="+fmt.Sprintf("%d", doer.ID), diff --git a/modules/private/hook.go b/modules/private/hook.go new file mode 100644 index 000000000000..2bc451307859 --- /dev/null +++ b/modules/private/hook.go @@ -0,0 +1,72 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package private + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "code.gitea.io/gitea/modules/setting" +) + +// HookOptions represents the options for the Hook calls +type HookOptions struct { + OldCommitID string + NewCommitID string + RefFullName string + UserID int64 + UserName string +} + +// HookPreReceive check whether the provided commits are allowed +func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d", + url.PathEscape(ownerName), + url.PathEscape(repoName), + url.QueryEscape(opts.OldCommitID), + url.QueryEscape(opts.NewCommitID), + url.QueryEscape(opts.RefFullName), + opts.UserID) + + resp, err := newInternalRequest(reqURL, "GET").Response() + if err != nil { + return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return resp.StatusCode, decodeJSONError(resp).Err + } + + return http.StatusOK, "" +} + +// HookPostReceive updates services and users +func HookPostReceive(ownerName, repoName string, opts HookOptions) (map[string]interface{}, string) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&username=%s", + url.PathEscape(ownerName), + url.PathEscape(repoName), + url.QueryEscape(opts.OldCommitID), + url.QueryEscape(opts.NewCommitID), + url.QueryEscape(opts.RefFullName), + opts.UserID, + url.QueryEscape(opts.UserName)) + + resp, err := newInternalRequest(reqURL, "GET").Response() + if err != nil { + return nil, fmt.Sprintf("Unable to contact gitea: %v", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, decodeJSONError(resp).Err + } + res := map[string]interface{}{} + _ = json.NewDecoder(resp.Body).Decode(&res) + + return res, "" +} diff --git a/routers/private/hook.go b/routers/private/hook.go new file mode 100644 index 000000000000..96aaa1d90d8c --- /dev/null +++ b/routers/private/hook.go @@ -0,0 +1,193 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package private includes all internal routes. The package name internal is ideal but Golang is not allowed, so we use private as package name instead. +package private + +import ( + "fmt" + "net/http" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" + macaron "gopkg.in/macaron.v1" +) + +// HookPreReceive checks whether a individual commit is acceptable +func HookPreReceive(ctx *macaron.Context) { + ownerName := ctx.Params(":owner") + repoName := ctx.Params(":repo") + oldCommitID := ctx.Query("old") + newCommitID := ctx.Query("new") + refFullName := ctx.Query("ref") + userID := ctx.QueryInt64("userID") + + branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) + repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Unable to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": err.Error(), + }) + return + } + protectBranch, err := models.GetProtectedBranchBy(repo.ID, branchName) + if err != nil { + log.Error("Unable to get protected branch: %s in %-v Error: %v", branchName, repo, err) + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + if protectBranch != nil && protectBranch.IsProtected() { + // check and deletion + if newCommitID == git.EmptySHA { + log.Warn("Forbidden: Branch: %s in %-v is protected from deletion", branchName, repo) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("branch %s is protected from deletion", branchName), + }) + return + } + + // detect force push + if git.EmptySHA != oldCommitID { + output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDir(repo.RepoPath()) + if err != nil { + log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Fail to detect force push: %v", err), + }) + return + } else if len(output) > 0 { + log.Warn("Forbidden: Branch: %s in %-v is protected from force push", branchName, repo) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("branch %s is protected from force push", branchName), + }) + return + + } + } + + if !protectBranch.CanUserPush(userID) { + log.Warn("Forbidden: User %d cannot push to protected branch: %s in %-v", userID, branchName, repo) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("protected branch %s can not be pushed to", branchName), + }) + return + } + } + ctx.PlainText(http.StatusOK, []byte("ok")) +} + +// HookPostReceive updates services and users +func HookPostReceive(ctx *macaron.Context) { + ownerName := ctx.Params(":owner") + repoName := ctx.Params(":repo") + oldCommitID := ctx.Query("old") + newCommitID := ctx.Query("new") + refFullName := ctx.Query("ref") + userID := ctx.QueryInt64("userID") + userName := ctx.Query("username") + + branch := refFullName + if strings.HasPrefix(refFullName, git.BranchPrefix) { + branch = strings.TrimPrefix(refFullName, git.BranchPrefix) + } else if strings.HasPrefix(refFullName, git.TagPrefix) { + branch = strings.TrimPrefix(refFullName, git.TagPrefix) + } + + // Only trigger activity updates for changes to branches or + // tags. Updates to other refs (eg, refs/notes, refs/changes, + // or other less-standard refs spaces are ignored since there + // may be a very large number of them). + if strings.HasPrefix(refFullName, git.BranchPrefix) || strings.HasPrefix(refFullName, git.TagPrefix) { + if err := models.PushUpdate(branch, models.PushUpdateOptions{ + RefFullName: refFullName, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + PusherID: userID, + PusherName: userName, + RepoUserName: ownerName, + RepoName: repoName, + }); err != nil { + log.Error("Failed to Update: %s/%s Branch: %s Error: %v", ownerName, repoName, branch, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Failed to Update: %s/%s Branch: %s Error: %v", ownerName, repoName, branch, err), + }) + return + } + } + + if newCommitID != git.EmptySHA && strings.HasPrefix(refFullName, git.BranchPrefix) { + repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) + if err != nil { + log.Error("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Failed to get repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + pullRequestAllowed := repo.AllowsPulls() + if !pullRequestAllowed { + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": false, + }) + } + + baseRepo := repo + if repo.IsFork { + if err := repo.GetBaseRepo(); err != nil { + log.Error("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf("Failed to get Base Repository of Forked repository: %-v Error: %v", repo, err), + }) + return + } + baseRepo = repo.BaseRepo + } + + if !repo.IsFork && branch == baseRepo.DefaultBranch { + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": false, + }) + } + + pr, err := models.GetUnmergedPullRequest(repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch) + if err != nil && !models.IsErrPullRequestNotExist(err) { + log.Error("Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": fmt.Sprintf( + "Failed to get active PR in: %-v Branch: %s to: %-v Branch: %s Error: %v", repo, branch, baseRepo, baseRepo.DefaultBranch, err), + }) + return + } + + if pr == nil { + if repo.IsFork { + branch = fmt.Sprintf("%s:%s", repo.OwnerName, branch) + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": true, + "create": true, + "branch": branch, + "url": fmt.Sprintf("%s/compare/%s...%s", baseRepo.HTMLURL(), util.PathEscapeSegments(baseRepo.DefaultBranch), util.PathEscapeSegments(branch)), + }) + } else { + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": true, + "create": false, + "branch": branch, + "url": fmt.Sprintf("%s/pulls/%d", baseRepo.HTMLURL(), pr.Index), + }) + } + return + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": false, + }) + return +} diff --git a/routers/private/internal.go b/routers/private/internal.go index ee6e1274c3e6..01ad5d2a5792 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -90,5 +90,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/branch/:id/*", GetProtectedBranchBy) m.Get("/repository/:rid", GetRepository) m.Get("/active-pull-request", GetActivePullRequest) + m.Get("/hook/pre-receive/:owner/:repo", HookPreReceive) + m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) }, CheckInternalToken) } From 04f50813e713a77ef3c1efe416b133fa32e1ed4c Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 19 May 2019 17:24:57 +0100 Subject: [PATCH 02/25] Internalise serv logic --- cmd/serv.go | 159 ++++++--------------- modules/private/hook.go | 2 +- modules/private/key.go | 20 +++ modules/private/serv.go | 106 ++++++++++++++ routers/private/hook.go | 3 + routers/private/internal.go | 3 + routers/private/key.go | 33 +++++ routers/private/serv.go | 275 ++++++++++++++++++++++++++++++++++++ 8 files changed, 482 insertions(+), 119 deletions(-) create mode 100644 modules/private/serv.go create mode 100644 routers/private/serv.go diff --git a/cmd/serv.go b/cmd/serv.go index a30e02e7a264..6ef12c6aa732 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -8,9 +8,12 @@ package cmd import ( "encoding/json" "fmt" + "net/http" + "net/url" "os" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -104,6 +107,7 @@ func fail(userMessage, logMessage string, args ...interface{}) { } func runServ(c *cli.Context) error { + // FIXME: This needs to internationalised setup("serv.log") if setting.SSH.Disabled { @@ -116,9 +120,23 @@ func runServ(c *cli.Context) error { return nil } + keys := strings.Split(c.Args()[0], "-") + if len(keys) != 2 || keys[0] != "key" { + fail("Key ID format error", "Invalid key argument: %s", c.Args()[0]) + } + keyID := com.StrTo(keys[1]).MustInt64() + cmd := os.Getenv("SSH_ORIGINAL_COMMAND") if len(cmd) == 0 { - println("Hi there, You've successfully authenticated, but Gitea does not provide shell access.") + key, user, err := private.ServNoCommand(keyID) + if err != nil { + fail("Internal error", "Failed to check provided key: %v", err) + } + if key.Type == models.KeyTypeDeploy { + println("Hi there! You've successfully authenticated with the deploy key named " + key.Name + ", but Gitea does not provide shell access.") + } else { + println("Hi there: " + user.Name + "! You've successfully authenticated with the key named " + key.Name + ", but Gitea does not provide shell access.") + } println("If this is unexpected, please log in with password and setup Gitea under another user.") return nil } @@ -159,34 +177,6 @@ func runServ(c *cli.Context) error { }() } - var ( - isWiki bool - unitType = models.UnitTypeCode - unitName = "code" - ) - if strings.HasSuffix(reponame, ".wiki") { - isWiki = true - unitType = models.UnitTypeWiki - unitName = "wiki" - reponame = reponame[:len(reponame)-5] - } - - os.Setenv(models.EnvRepoUsername, username) - if isWiki { - os.Setenv(models.EnvRepoIsWiki, "true") - } else { - os.Setenv(models.EnvRepoIsWiki, "false") - } - os.Setenv(models.EnvRepoName, reponame) - - repo, err := private.GetRepositoryByOwnerAndName(username, reponame) - if err != nil { - if strings.Contains(err.Error(), "Failed to get repository: repository does not exist") { - fail(accessDenied, "Repository does not exist: %s/%s", username, reponame) - } - fail("Internal error", "Failed to get repository: %v", err) - } - requestedMode, has := allowedCommands[verb] if !has { fail("Unknown git command", "Unknown git command %s", verb) @@ -202,97 +192,36 @@ func runServ(c *cli.Context) error { } } - // Prohibit push to mirror repositories. - if requestedMode > models.AccessModeRead && repo.IsMirror { - fail("mirror repository is read-only", "") - } - - // Allow anonymous clone for public repositories. - var ( - keyID int64 - user *models.User - ) - if requestedMode == models.AccessModeWrite || repo.IsPrivate || setting.Service.RequireSignInView { - keys := strings.Split(c.Args()[0], "-") - if len(keys) != 2 { - fail("Key ID format error", "Invalid key argument: %s", c.Args()[0]) - } - - key, err := private.GetPublicKeyByID(com.StrTo(keys[1]).MustInt64()) - if err != nil { - fail("Invalid key ID", "Invalid key ID[%s]: %v", c.Args()[0], err) - } - keyID = key.ID - - // Check deploy key or user key. - if key.Type == models.KeyTypeDeploy { - // Now we have to get the deploy key for this repo - deployKey, err := private.GetDeployKey(key.ID, repo.ID) - if err != nil { - fail("Key access denied", "Failed to access internal api: [key_id: %d, repo_id: %d]", key.ID, repo.ID) - } - - if deployKey == nil { - fail("Key access denied", "Deploy key access denied: [key_id: %d, repo_id: %d]", key.ID, repo.ID) - } - - if deployKey.Mode < requestedMode { - fail("Key permission denied", "Cannot push with read-only deployment key: %d to repo_id: %d", key.ID, repo.ID) - } - - // Update deploy key activity. - if err = private.UpdateDeployKeyUpdated(key.ID, repo.ID); err != nil { - fail("Internal error", "UpdateDeployKey: %v", err) - } - - // FIXME: Deploy keys aren't really the owner of the repo pushing changes - // however we don't have good way of representing deploy keys in hook.go - // so for now use the owner - os.Setenv(models.EnvPusherName, username) - os.Setenv(models.EnvPusherID, fmt.Sprintf("%d", repo.OwnerID)) - } else { - user, err = private.GetUserByKeyID(key.ID) - if err != nil { - fail("internal error", "Failed to get user by key ID(%d): %v", keyID, err) - } - - if !user.IsActive || user.ProhibitLogin { - fail("Your account is not active or has been disabled by Administrator", - "User %s is disabled and have no access to repository %s", - user.Name, repoPath) - } - - mode, err := private.CheckUnitUser(user.ID, repo.ID, user.IsAdmin, unitType) - if err != nil { - fail("Internal error", "Failed to check access: %v", err) - } else if *mode < requestedMode { - clientMessage := accessDenied - if *mode >= models.AccessModeRead { - clientMessage = "You do not have sufficient authorization for this action" - } - fail(clientMessage, - "User %s does not have level %v access to repository %s's "+unitName, - user.Name, requestedMode, repoPath) + results, err := private.ServCommand(keyID, username, reponame, requestedMode, verb, lfsVerb) + if err != nil { + if private.IsErrServCommand(err) { + errServCommand := err.(private.ErrServCommand) + if errServCommand.StatusCode != http.StatusInternalServerError { + fail("Unauthorized", errServCommand.Error()) + } else { + fail("Internal Server Error", errServCommand.Error()) } - - os.Setenv(models.EnvPusherName, user.Name) - os.Setenv(models.EnvPusherID, fmt.Sprintf("%d", user.ID)) } + fail("Internal Server Error", err.Error()) } + os.Setenv(models.EnvRepoIsWiki, strconv.FormatBool(results.IsWiki)) + os.Setenv(models.EnvRepoName, results.RepoName) + os.Setenv(models.EnvRepoUsername, results.OwnerName) + os.Setenv(models.EnvPusherName, username) + os.Setenv(models.EnvPusherID, strconv.FormatInt(results.UserID, 10)) + os.Setenv(models.ProtectedBranchRepoID, strconv.FormatInt(results.RepoID, 10)) //LFS token authentication if verb == lfsAuthenticateVerb { - url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, username, repo.Name) + url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName)) now := time.Now() claims := jwt.MapClaims{ - "repo": repo.ID, + "repo": results.RepoID, "op": lfsVerb, "exp": now.Add(setting.LFS.HTTPAuthExpiry).Unix(), "nbf": now.Unix(), - } - if user != nil { - claims["user"] = user.ID + "user": results.UserID, } token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) @@ -313,7 +242,6 @@ func runServ(c *cli.Context) error { if err != nil { fail("Internal error", "Failed to encode LFS json response: %v", err) } - return nil } @@ -329,13 +257,8 @@ func runServ(c *cli.Context) error { } else { gitcmd = exec.Command(verb, repoPath) } - if isWiki { - if err = private.InitWiki(repo.ID); err != nil { - fail("Internal error", "Failed to init wiki repo: %v", err) - } - } - os.Setenv(models.ProtectedBranchRepoID, fmt.Sprintf("%d", repo.ID)) + os.Setenv(models.ProtectedBranchRepoID, fmt.Sprintf("%d", results.RepoID)) gitcmd.Dir = setting.RepoRootPath gitcmd.Stdout = os.Stdout @@ -346,9 +269,9 @@ func runServ(c *cli.Context) error { } // Update user key activity. - if keyID > 0 { - if err = private.UpdatePublicKeyUpdated(keyID); err != nil { - fail("Internal error", "UpdatePublicKey: %v", err) + if results.KeyID > 0 { + if err = private.UpdatePublicKeyInRepo(results.KeyID, results.RepoID); err != nil { + fail("Internal error", "UpdatePublicKeyInRepo: %v", err) } } diff --git a/modules/private/hook.go b/modules/private/hook.go index 2bc451307859..69729b8c32d7 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/modules/private/key.go b/modules/private/key.go index 1c6511846b77..de6b4f31c1d6 100644 --- a/modules/private/key.go +++ b/modules/private/key.go @@ -139,3 +139,23 @@ func UpdatePublicKeyUpdated(keyID int64) error { } return nil } + +// UpdatePublicKeyInRepo update public key and if necessary deploy key updates +func UpdatePublicKeyInRepo(keyID, repoID int64) error { + // Ask for running deliver hook and test pull request tasks. + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d/update/%d", keyID, repoID) + log.GitLogger.Trace("UpdatePublicKeyUpdated: %s", reqURL) + + resp, err := newInternalRequest(reqURL, "POST").Response() + if err != nil { + return err + } + + defer resp.Body.Close() + + // All 2XX status codes are accepted and others will return an error + if resp.StatusCode/100 != 2 { + return fmt.Errorf("Failed to update public key: %s", decodeJSONError(resp).Err) + } + return nil +} diff --git a/modules/private/serv.go b/modules/private/serv.go new file mode 100644 index 000000000000..5b4a27f11621 --- /dev/null +++ b/modules/private/serv.go @@ -0,0 +1,106 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package private + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" +) + +// KeyAndOwner is the response from ServNoCommand +type KeyAndOwner struct { + Key *models.PublicKey `json:"key"` + Owner *models.User `json:"user"` +} + +// ServNoCommand returns information about the provided key +func ServNoCommand(keyID int64) (*models.PublicKey, *models.User, error) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/none/%d", + keyID) + resp, err := newInternalRequest(reqURL, "GET").Response() + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("%s", decodeJSONError(resp).Err) + } + + var keyAndOwner KeyAndOwner + if err := json.NewDecoder(resp.Body).Decode(&keyAndOwner); err != nil { + return nil, nil, err + } + return keyAndOwner.Key, keyAndOwner.Owner, nil +} + +// ServCommandResults are the results of a call to the private route serv +type ServCommandResults struct { + IsWiki bool + IsDeployKey bool + KeyID int64 + KeyName string + UserName string + UserID int64 + OwnerName string + RepoName string + RepoID int64 +} + +// ErrServCommand is an error returned from ServCommmand. +type ErrServCommand struct { + Results ServCommandResults + Type string + Err string + StatusCode int +} + +func (err ErrServCommand) Error() string { + return err.Err +} + +// IsErrServCommand checks if an error is a ErrServCommand. +func IsErrServCommand(err error) bool { + _, ok := err.(ErrServCommand) + return ok +} + +// ServCommand preps for a serv call +func ServCommand(keyID int64, ownerName, repoName string, mode models.AccessMode, verbs ...string) (*ServCommandResults, error) { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/command/%d/%s/%s?mode=%d", + keyID, + url.PathEscape(ownerName), + url.PathEscape(repoName), + mode) + for _, verb := range verbs { + if verb != "" { + reqURL += fmt.Sprintf("&verb=%s", url.QueryEscape(verb)) + } + } + + resp, err := newInternalRequest(reqURL, "GET").Response() + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + var errServCommand ErrServCommand + if err := json.NewDecoder(resp.Body).Decode(&errServCommand); err != nil { + return nil, err + } + errServCommand.StatusCode = resp.StatusCode + return nil, errServCommand + } + var results ServCommandResults + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return nil, err + } + return &results, nil + +} diff --git a/routers/private/hook.go b/routers/private/hook.go index 96aaa1d90d8c..a2a12f54c2a4 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -35,6 +35,7 @@ func HookPreReceive(ctx *macaron.Context) { }) return } + repo.OwnerName = ownerName protectBranch, err := models.GetProtectedBranchBy(repo.ID, branchName) if err != nil { log.Error("Unable to get protected branch: %s in %-v Error: %v", branchName, repo, err) @@ -131,6 +132,8 @@ func HookPostReceive(ctx *macaron.Context) { }) return } + repo.OwnerName = ownerName + pullRequestAllowed := repo.AllowsPulls() if !pullRequestAllowed { ctx.JSON(http.StatusOK, map[string]interface{}{ diff --git a/routers/private/internal.go b/routers/private/internal.go index 01ad5d2a5792..f8a8e95a4a2f 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -79,6 +79,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/ssh/:id", GetPublicKeyByID) m.Get("/ssh/:id/user", GetUserByKeyID) m.Post("/ssh/:id/update", UpdatePublicKey) + m.Post("/ssh/:id/update/:repoid", UpdatePublicKeyInRepo) m.Post("/repositories/:repoid/keys/:keyid/update", UpdateDeployKey) m.Get("/repositories/:repoid/user/:userid/checkunituser", CheckUnitUser) m.Get("/repositories/:repoid/has-keys/:keyid", HasDeployKey) @@ -92,5 +93,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/active-pull-request", GetActivePullRequest) m.Get("/hook/pre-receive/:owner/:repo", HookPreReceive) m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) + m.Get("/serv/none/:keyid", ServNoCommand) + m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) }, CheckInternalToken) } diff --git a/routers/private/key.go b/routers/private/key.go index ee22f6ac4881..db0a725eac4d 100644 --- a/routers/private/key.go +++ b/routers/private/key.go @@ -46,6 +46,39 @@ func UpdatePublicKey(ctx *macaron.Context) { ctx.PlainText(200, []byte("success")) } +// UpdatePublicKeyInRepo update public key and deploy key updates +func UpdatePublicKeyInRepo(ctx *macaron.Context) { + keyID := ctx.ParamsInt64(":id") + repoID := ctx.ParamsInt64(":repoid") + if err := models.UpdatePublicKeyUpdated(keyID); err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + deployKey, err := models.GetDeployKeyByRepo(keyID, repoID) + if err != nil { + if models.IsErrDeployKeyNotExist(err) { + ctx.PlainText(200, []byte("success")) + return + } + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + deployKey.UpdatedUnix = util.TimeStampNow() + if err = models.UpdateDeployKeyCols(deployKey, "updated_unix"); err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + ctx.PlainText(200, []byte("success")) +} + //GetPublicKeyByID chainload to models.GetPublicKeyByID func GetPublicKeyByID(ctx *macaron.Context) { keyID := ctx.ParamsInt64(":id") diff --git a/routers/private/serv.go b/routers/private/serv.go new file mode 100644 index 000000000000..3b8c6cab4060 --- /dev/null +++ b/routers/private/serv.go @@ -0,0 +1,275 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package private includes all internal routes. The package name internal is ideal but Golang is not allowed, so we use private as package name instead. +package private + +import ( + "fmt" + "net/http" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/private" + "code.gitea.io/gitea/modules/setting" + macaron "gopkg.in/macaron.v1" +) + +// ServNoCommand returns information about the provided keyid +func ServNoCommand(ctx *macaron.Context) { + keyID := ctx.ParamsInt64(":keyid") + if keyID <= 0 { + ctx.JSON(http.StatusBadRequest, map[string]interface{}{ + "err": fmt.Sprintf("Bad key id: %d", keyID), + }) + } + results := private.KeyAndOwner{} + + key, err := models.GetPublicKeyByID(keyID) + if err != nil { + if models.IsErrKeyNotExist(err) { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "err": fmt.Sprintf("Cannot find key: %d", keyID), + }) + return + } + log.Error("Unable to get public key: %d Error: %v", keyID, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": err.Error(), + }) + return + } + results.Key = key + + if key.Type == models.KeyTypeUser { + user, err := models.GetUserByID(key.OwnerID) + if err != nil { + if models.IsErrUserNotExist(err) { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "err": fmt.Sprintf("Cannot find owner with id: %d for key: %d", key.OwnerID, keyID), + }) + return + } + log.Error("Unable to get owner with id: %d for public key: %d Error: %v", key.OwnerID, keyID, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": err.Error(), + }) + return + } + results.Owner = user + } + ctx.JSON(http.StatusOK, &results) + return +} + +// ServCommand returns information about the provided keyid +func ServCommand(ctx *macaron.Context) { + // Although we provide the verbs we don't need them at present they're just for logging purposes + keyID := ctx.ParamsInt64(":keyid") + ownerName := ctx.Params(":owner") + repoName := ctx.Params(":repo") + mode := models.AccessMode(ctx.QueryInt("mode")) + + // Set the basic parts of the results to return + results := private.ServCommandResults{ + RepoName: repoName, + OwnerName: ownerName, + KeyID: keyID, + } + + // Now because we're not translating things properly let's just default some Engish strings here + modeString := "read" + if mode > models.AccessModeRead { + modeString = "write to" + } + + // The default unit we're trying to look at is code + unitType := models.UnitTypeCode + + // Unless we're a wiki... + if strings.HasSuffix(repoName, ".wiki") { + // in which case we need to look at the wiki + unitType = models.UnitTypeWiki + // And we'd better munge the reponame and tell downstream we're looking at a wiki + results.IsWiki = true + results.RepoName = repoName[:len(repoName)-5] + } + + // Now get the Repository and set the results section + repo, err := models.GetRepositoryByOwnerAndName(results.OwnerName, results.RepoName) + if err != nil { + if models.IsErrRepoNotExist(err) { + ctx.JSON(http.StatusNotFound, map[string]interface{}{ + "results": results, + "type": "ErrRepoNotExist", + "err": fmt.Sprintf("Cannot find repository %s/%s", results.OwnerName, results.RepoName), + }) + return + } + log.Error("Unable to get repository: %s/%s Error: %v", results.OwnerName, results.RepoName, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": fmt.Sprintf("Unable to get repository: %s/%s %v", results.OwnerName, results.RepoName, err), + }) + return + } + repo.OwnerName = ownerName + results.RepoID = repo.ID + + // We can shortcut at this point if the repo is a mirror + if mode > models.AccessModeRead && repo.IsMirror { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "results": results, + "type": "ErrMirrorReadOnly", + "err": fmt.Sprintf("Mirror Repository %s/%s is read-only", results.OwnerName, results.RepoName), + }) + return + } + + // Get the Public Key represented by the keyID + key, err := models.GetPublicKeyByID(keyID) + if err != nil { + if models.IsErrKeyNotExist(err) { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "results": results, + "type": "ErrKeyNotExist", + "err": fmt.Sprintf("Cannot find key: %d", keyID), + }) + return + } + log.Error("Unable to get public key: %d Error: %v", keyID, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": fmt.Sprintf("Unable to get key: %d Error: %v", keyID, err), + }) + return + } + results.KeyName = key.Name + results.KeyID = key.ID + results.UserID = key.OwnerID + + // Deploy Keys have ownerID set to 0 therefore we can't use the owner + // So now we need to check if the key is a deploy key + // We'll keep hold of the deploy key here for permissions checking + var deployKey *models.DeployKey + var user *models.User + if key.Type == models.KeyTypeDeploy { + results.IsDeployKey = true + + var err error + deployKey, err = models.GetDeployKeyByRepo(key.ID, repo.ID) + if err != nil { + if models.IsErrDeployKeyNotExist(err) { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "results": results, + "type": "ErrDeployKeyNotExist", + "err": fmt.Sprintf("Public (Deploy) Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, results.OwnerName, results.RepoName), + }) + return + } + log.Error("Unable to get deploy for public (deploy) key: %d in %-v Error: %v", key.ID, repo, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": fmt.Sprintf("Unable to get Deploy Key for Public Key: %d:%s in %s/%s.", key.ID, key.Name, results.OwnerName, results.RepoName), + }) + return + } + results.KeyName = deployKey.Name + + // FIXME: Deploy keys aren't really the owner of the repo pushing changes + // however we don't have good way of representing deploy keys in hook.go + // so for now use the owner of the repository + results.UserName = results.OwnerName + results.UserID = repo.OwnerID + } else { + // Get the user represented by the Key + var err error + user, err = models.GetUserByID(key.OwnerID) + if err != nil { + if models.IsErrUserNotExist(err) { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "results": results, + "type": "ErrUserNotExist", + "err": fmt.Sprintf("Public Key: %d:%s owner %d does not exist.", key.ID, key.Name, key.OwnerID), + }) + return + } + log.Error("Unable to get owner: %d for public key: %d:%s Error: %v", key.OwnerID, key.ID, key.Name, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": fmt.Sprintf("Unable to get Owner: %d for Deploy Key: %d:%s in %s/%s.", key.OwnerID, key.ID, key.Name, ownerName, repoName), + }) + return + } + results.UserName = user.Name + } + + // Permissions checking: + if mode > models.AccessModeRead || repo.IsPrivate || setting.Service.RequireSignInView { + if key.Type == models.KeyTypeDeploy { + if deployKey.Mode < mode { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "results": results, + "type": "ErrUnauthorized", + "err": fmt.Sprintf("Deploy Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, ownerName, repoName), + }) + return + } + } else { + perm, err := models.GetUserRepoPermission(repo, user) + if err != nil { + log.Error("Unable to get permissions for %-v with key %d in %-v Error: %v", user, key.ID, repo, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": fmt.Sprintf("Unable to get permissions for user %d:%s with key %d in %s/%s Error: %v", user.ID, user.Name, key.ID, ownerName, repoName, err), + }) + return + } + + userMode := perm.UnitAccessMode(unitType) + + if userMode < mode { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "results": results, + "type": "ErrUnauthorized", + "err": fmt.Sprintf("User: %d:%s with Key: %d:%s is not authorized to %s %s/%s.", user.ID, user.Name, key.ID, key.Name, modeString, ownerName, repoName), + }) + return + } + } + } + + // Finally if we're trying to touch the wiki we should init it + if results.IsWiki { + if err = repo.InitWiki(); err != nil { + log.Error("Failed to initialize the wiki in %-v Error: %v", repo, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": fmt.Sprintf("Failed to initialize the wiki in %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + } + log.Debug("Serv Results:\nIsWiki: %t\nIsDeployKey: %t\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", + results.IsWiki, + results.IsDeployKey, + results.KeyID, + results.KeyName, + results.UserName, + results.UserID, + results.OwnerName, + results.RepoName, + results.RepoID) + + ctx.JSON(http.StatusOK, results) + // We will update the keys in a different call. + return +} From 997f16dd050dacbfe79742e0372dbfc211369de0 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 19 May 2019 19:53:50 +0100 Subject: [PATCH 03/25] Remove old internal paths --- integrations/internal_test.go | 44 ----------- modules/private/branch.go | 67 ----------------- modules/private/internal.go | 49 ------------ modules/private/key.go | 134 +-------------------------------- modules/private/push_update.go | 40 ---------- modules/private/repository.go | 68 ----------------- modules/private/wiki.go | 33 -------- routers/private/branch.go | 52 ------------- routers/private/internal.go | 14 ---- routers/private/key.go | 89 ---------------------- routers/private/push_update.go | 47 ------------ routers/private/repository.go | 83 -------------------- routers/private/wiki.go | 34 --------- 13 files changed, 1 insertion(+), 753 deletions(-) delete mode 100644 integrations/internal_test.go delete mode 100644 modules/private/branch.go delete mode 100644 modules/private/push_update.go delete mode 100644 modules/private/repository.go delete mode 100644 modules/private/wiki.go delete mode 100644 routers/private/branch.go delete mode 100644 routers/private/push_update.go delete mode 100644 routers/private/repository.go delete mode 100644 routers/private/wiki.go diff --git a/integrations/internal_test.go b/integrations/internal_test.go deleted file mode 100644 index ee0c0d18f157..000000000000 --- a/integrations/internal_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package integrations - -import ( - "encoding/json" - "fmt" - "net/http" - "testing" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" - - "github.com/stretchr/testify/assert" -) - -func assertProtectedBranch(t *testing.T, repoID int64, branchName string, isErr, canPush bool) { - reqURL := fmt.Sprintf("/api/internal/branch/%d/%s", repoID, util.PathEscapeSegments(branchName)) - req := NewRequest(t, "GET", reqURL) - t.Log(reqURL) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", setting.InternalToken)) - - resp := MakeRequest(t, req, NoExpectedStatus) - if isErr { - assert.EqualValues(t, http.StatusInternalServerError, resp.Code) - } else { - assert.EqualValues(t, http.StatusOK, resp.Code) - var branch models.ProtectedBranch - t.Log(resp.Body.String()) - assert.NoError(t, json.Unmarshal(resp.Body.Bytes(), &branch)) - assert.Equal(t, canPush, !branch.IsProtected()) - } -} - -func TestInternal_GetProtectedBranch(t *testing.T) { - prepareTestEnv(t) - - assertProtectedBranch(t, 1, "master", false, true) - assertProtectedBranch(t, 1, "dev", false, true) - assertProtectedBranch(t, 1, "lunny/dev", false, true) -} diff --git a/modules/private/branch.go b/modules/private/branch.go deleted file mode 100644 index bbd0d4b69738..000000000000 --- a/modules/private/branch.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "encoding/json" - "fmt" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" -) - -// GetProtectedBranchBy get protected branch information -func GetProtectedBranchBy(repoID int64, branchName string) (*models.ProtectedBranch, error) { - // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/branch/%d/%s", repoID, util.PathEscapeSegments(branchName)) - log.GitLogger.Trace("GetProtectedBranchBy: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, err - } - - var branch models.ProtectedBranch - if err := json.NewDecoder(resp.Body).Decode(&branch); err != nil { - return nil, err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return nil, fmt.Errorf("Failed to get protected branch: %s", decodeJSONError(resp).Err) - } - - return &branch, nil -} - -// CanUserPush returns if user can push -func CanUserPush(protectedBranchID, userID int64) (bool, error) { - // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/protectedbranch/%d/%d", protectedBranchID, userID) - log.GitLogger.Trace("CanUserPush: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return false, err - } - - var canPush = make(map[string]interface{}) - if err := json.NewDecoder(resp.Body).Decode(&canPush); err != nil { - return false, err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return false, fmt.Errorf("Failed to retrieve push user: %s", decodeJSONError(resp).Err) - } - - return canPush["can_push"].(bool), nil -} diff --git a/modules/private/internal.go b/modules/private/internal.go index 56852ce63c11..b4fee2680fba 100644 --- a/modules/private/internal.go +++ b/modules/private/internal.go @@ -10,11 +10,8 @@ import ( "fmt" "net" "net/http" - "net/url" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/httplib" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) @@ -51,49 +48,3 @@ func newInternalRequest(url, method string) *httplib.Request { } return req } - -// CheckUnitUser check whether user could visit the unit of this repository -func CheckUnitUser(userID, repoID int64, isAdmin bool, unitType models.UnitType) (*models.AccessMode, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/repositories/%d/user/%d/checkunituser?isAdmin=%t&unitType=%d", repoID, userID, isAdmin, unitType) - log.GitLogger.Trace("CheckUnitUser: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("Failed to CheckUnitUser: %s", decodeJSONError(resp).Err) - } - - var a models.AccessMode - if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { - return nil, err - } - - return &a, nil -} - -// GetRepositoryByOwnerAndName returns the repository by given ownername and reponame. -func GetRepositoryByOwnerAndName(ownerName, repoName string) (*models.Repository, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/repo/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - log.GitLogger.Trace("GetRepositoryByOwnerAndName: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("Failed to get repository: %s", decodeJSONError(resp).Err) - } - - var repo models.Repository - if err := json.NewDecoder(resp.Body).Decode(&repo); err != nil { - return nil, err - } - - return &repo, nil -} diff --git a/modules/private/key.go b/modules/private/key.go index de6b4f31c1d6..f81dd24d9084 100644 --- a/modules/private/key.go +++ b/modules/private/key.go @@ -1,151 +1,19 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package private import ( - "encoding/json" "fmt" - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) -// UpdateDeployKeyUpdated update deploy key updates -func UpdateDeployKeyUpdated(keyID int64, repoID int64) error { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/repositories/%d/keys/%d/update", repoID, keyID) - log.GitLogger.Trace("UpdateDeployKeyUpdated: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "POST").Response() - if err != nil { - return err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return fmt.Errorf("Failed to update deploy key: %s", decodeJSONError(resp).Err) - } - return nil -} - -// GetDeployKey check if repo has deploy key -func GetDeployKey(keyID, repoID int64) (*models.DeployKey, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/repositories/%d/keys/%d", repoID, keyID) - log.GitLogger.Trace("GetDeployKey: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, err - } - defer resp.Body.Close() - - switch resp.StatusCode { - case 404: - return nil, nil - case 200: - var dKey models.DeployKey - if err := json.NewDecoder(resp.Body).Decode(&dKey); err != nil { - return nil, err - } - return &dKey, nil - default: - return nil, fmt.Errorf("Failed to get deploy key: %s", decodeJSONError(resp).Err) - } -} - -// HasDeployKey check if repo has deploy key -func HasDeployKey(keyID, repoID int64) (bool, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/repositories/%d/has-keys/%d", repoID, keyID) - log.GitLogger.Trace("HasDeployKey: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return false, err - } - defer resp.Body.Close() - - if resp.StatusCode == 200 { - return true, nil - } - return false, nil -} - -// GetPublicKeyByID get public ssh key by his ID -func GetPublicKeyByID(keyID int64) (*models.PublicKey, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d", keyID) - log.GitLogger.Trace("GetPublicKeyByID: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("Failed to get repository: %s", decodeJSONError(resp).Err) - } - - var pKey models.PublicKey - if err := json.NewDecoder(resp.Body).Decode(&pKey); err != nil { - return nil, err - } - return &pKey, nil -} - -// GetUserByKeyID get user attached to key -func GetUserByKeyID(keyID int64) (*models.User, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d/user", keyID) - log.GitLogger.Trace("GetUserByKeyID: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("Failed to get user: %s", decodeJSONError(resp).Err) - } - - var user models.User - if err := json.NewDecoder(resp.Body).Decode(&user); err != nil { - return nil, err - } - - return &user, nil -} - -// UpdatePublicKeyUpdated update public key updates -func UpdatePublicKeyUpdated(keyID int64) error { - // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d/update", keyID) - log.GitLogger.Trace("UpdatePublicKeyUpdated: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "POST").Response() - if err != nil { - return err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return fmt.Errorf("Failed to update public key: %s", decodeJSONError(resp).Err) - } - return nil -} - // UpdatePublicKeyInRepo update public key and if necessary deploy key updates func UpdatePublicKeyInRepo(keyID, repoID int64) error { // Ask for running deliver hook and test pull request tasks. reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d/update/%d", keyID, repoID) - log.GitLogger.Trace("UpdatePublicKeyUpdated: %s", reqURL) - resp, err := newInternalRequest(reqURL, "POST").Response() if err != nil { return err diff --git a/modules/private/push_update.go b/modules/private/push_update.go deleted file mode 100644 index f3071b63ade1..000000000000 --- a/modules/private/push_update.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "encoding/json" - "fmt" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -// PushUpdate update publick key updates -func PushUpdate(opt models.PushUpdateOptions) error { - // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + "api/internal/push/update" - log.GitLogger.Trace("PushUpdate: %s", reqURL) - - body, err := json.Marshal(&opt) - if err != nil { - return err - } - - resp, err := newInternalRequest(reqURL, "POST").Body(body).Response() - if err != nil { - return err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return fmt.Errorf("Failed to update public key: %s", decodeJSONError(resp).Err) - } - - return nil -} diff --git a/modules/private/repository.go b/modules/private/repository.go deleted file mode 100644 index cf8ae6840905..000000000000 --- a/modules/private/repository.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "encoding/json" - "fmt" - "net/url" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -// GetRepository return the repository by its ID and a bool about if it's allowed to have PR -func GetRepository(repoID int64) (*models.Repository, bool, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/repository/%d", repoID) - log.GitLogger.Trace("GetRepository: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, false, err - } - - var repoInfo struct { - Repository *models.Repository - AllowPullRequest bool - } - if err := json.NewDecoder(resp.Body).Decode(&repoInfo); err != nil { - return nil, false, err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return nil, false, fmt.Errorf("failed to retrieve repository: %s", decodeJSONError(resp).Err) - } - - return repoInfo.Repository, repoInfo.AllowPullRequest, nil -} - -// ActivePullRequest returns an active pull request if it exists -func ActivePullRequest(baseRepoID int64, headRepoID int64, baseBranch, headBranch string) (*models.PullRequest, error) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/active-pull-request?baseRepoID=%d&headRepoID=%d&baseBranch=%s&headBranch=%s", baseRepoID, headRepoID, url.QueryEscape(baseBranch), url.QueryEscape(headBranch)) - log.GitLogger.Trace("ActivePullRequest: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return nil, err - } - - var pr *models.PullRequest - if err := json.NewDecoder(resp.Body).Decode(&pr); err != nil { - return nil, err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return nil, fmt.Errorf("failed to retrieve pull request: %s", decodeJSONError(resp).Err) - } - - return pr, nil -} diff --git a/modules/private/wiki.go b/modules/private/wiki.go deleted file mode 100644 index 4ad0cc7c4ef5..000000000000 --- a/modules/private/wiki.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "fmt" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -// InitWiki initwiki via repo id -func InitWiki(repoID int64) error { - // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/repositories/%d/wiki/init", repoID) - log.GitLogger.Trace("InitWiki: %s", reqURL) - - resp, err := newInternalRequest(reqURL, "GET").Response() - if err != nil { - return err - } - - defer resp.Body.Close() - - // All 2XX status codes are accepted and others will return an error - if resp.StatusCode/100 != 2 { - return fmt.Errorf("Failed to init wiki: %s", decodeJSONError(resp).Err) - } - - return nil -} diff --git a/routers/private/branch.go b/routers/private/branch.go deleted file mode 100644 index 448c61f1dbba..000000000000 --- a/routers/private/branch.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "code.gitea.io/gitea/models" - - macaron "gopkg.in/macaron.v1" -) - -// GetProtectedBranchBy get protected branch information -func GetProtectedBranchBy(ctx *macaron.Context) { - repoID := ctx.ParamsInt64(":id") - branchName := ctx.Params("*") - protectBranch, err := models.GetProtectedBranchBy(repoID, branchName) - if err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } else if protectBranch != nil { - ctx.JSON(200, protectBranch) - } else { - ctx.JSON(200, &models.ProtectedBranch{ - ID: 0, - }) - } -} - -// CanUserPush returns if user push -func CanUserPush(ctx *macaron.Context) { - pbID := ctx.ParamsInt64(":pbid") - userID := ctx.ParamsInt64(":userid") - - protectBranch, err := models.GetProtectedBranchByID(pbID) - if err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } else if protectBranch != nil { - ctx.JSON(200, map[string]interface{}{ - "can_push": protectBranch.CanUserPush(userID), - }) - } else { - ctx.JSON(200, map[string]interface{}{ - "can_push": false, - }) - } -} diff --git a/routers/private/internal.go b/routers/private/internal.go index f8a8e95a4a2f..11cea8b4b9f3 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -76,21 +76,7 @@ func CheckUnitUser(ctx *macaron.Context) { // These APIs will be invoked by internal commands for example `gitea serv` and etc. func RegisterRoutes(m *macaron.Macaron) { m.Group("/", func() { - m.Get("/ssh/:id", GetPublicKeyByID) - m.Get("/ssh/:id/user", GetUserByKeyID) - m.Post("/ssh/:id/update", UpdatePublicKey) m.Post("/ssh/:id/update/:repoid", UpdatePublicKeyInRepo) - m.Post("/repositories/:repoid/keys/:keyid/update", UpdateDeployKey) - m.Get("/repositories/:repoid/user/:userid/checkunituser", CheckUnitUser) - m.Get("/repositories/:repoid/has-keys/:keyid", HasDeployKey) - m.Get("/repositories/:repoid/keys/:keyid", GetDeployKey) - m.Get("/repositories/:repoid/wiki/init", InitWiki) - m.Post("/push/update", PushUpdate) - m.Get("/protectedbranch/:pbid/:userid", CanUserPush) - m.Get("/repo/:owner/:repo", GetRepositoryByOwnerAndName) - m.Get("/branch/:id/*", GetProtectedBranchBy) - m.Get("/repository/:rid", GetRepository) - m.Get("/active-pull-request", GetActivePullRequest) m.Get("/hook/pre-receive/:owner/:repo", HookPreReceive) m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) m.Get("/serv/none/:keyid", ServNoCommand) diff --git a/routers/private/key.go b/routers/private/key.go index db0a725eac4d..f7212ec8929f 100644 --- a/routers/private/key.go +++ b/routers/private/key.go @@ -12,40 +12,6 @@ import ( macaron "gopkg.in/macaron.v1" ) -// UpdateDeployKey update deploy key updates -func UpdateDeployKey(ctx *macaron.Context) { - repoID := ctx.ParamsInt64(":repoid") - keyID := ctx.ParamsInt64(":keyid") - deployKey, err := models.GetDeployKeyByRepo(keyID, repoID) - if err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - deployKey.UpdatedUnix = util.TimeStampNow() - if err = models.UpdateDeployKeyCols(deployKey, "updated_unix"); err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - ctx.PlainText(200, []byte("success")) -} - -// UpdatePublicKey update publick key updates -func UpdatePublicKey(ctx *macaron.Context) { - keyID := ctx.ParamsInt64(":id") - if err := models.UpdatePublicKeyUpdated(keyID); err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - - ctx.PlainText(200, []byte("success")) -} - // UpdatePublicKeyInRepo update public key and deploy key updates func UpdatePublicKeyInRepo(ctx *macaron.Context) { keyID := ctx.ParamsInt64(":id") @@ -78,58 +44,3 @@ func UpdatePublicKeyInRepo(ctx *macaron.Context) { ctx.PlainText(200, []byte("success")) } - -//GetPublicKeyByID chainload to models.GetPublicKeyByID -func GetPublicKeyByID(ctx *macaron.Context) { - keyID := ctx.ParamsInt64(":id") - key, err := models.GetPublicKeyByID(keyID) - if err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - ctx.JSON(200, key) -} - -//GetUserByKeyID chainload to models.GetUserByKeyID -func GetUserByKeyID(ctx *macaron.Context) { - keyID := ctx.ParamsInt64(":id") - user, err := models.GetUserByKeyID(keyID) - if err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - ctx.JSON(200, user) -} - -//GetDeployKey chainload to models.GetDeployKey -func GetDeployKey(ctx *macaron.Context) { - repoID := ctx.ParamsInt64(":repoid") - keyID := ctx.ParamsInt64(":keyid") - dKey, err := models.GetDeployKeyByRepo(keyID, repoID) - if err != nil { - if models.IsErrDeployKeyNotExist(err) { - ctx.JSON(404, []byte("not found")) - return - } - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - ctx.JSON(200, dKey) -} - -//HasDeployKey chainload to models.HasDeployKey -func HasDeployKey(ctx *macaron.Context) { - repoID := ctx.ParamsInt64(":repoid") - keyID := ctx.ParamsInt64(":keyid") - if models.HasDeployKey(keyID, repoID) { - ctx.PlainText(200, []byte("success")) - return - } - ctx.PlainText(404, []byte("not found")) -} diff --git a/routers/private/push_update.go b/routers/private/push_update.go deleted file mode 100644 index 5c42f066ee7d..000000000000 --- a/routers/private/push_update.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "encoding/json" - "strings" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" - - macaron "gopkg.in/macaron.v1" -) - -// PushUpdate update public key updates -func PushUpdate(ctx *macaron.Context) { - var opt models.PushUpdateOptions - if err := json.NewDecoder(ctx.Req.Request.Body).Decode(&opt); err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - - branch := strings.TrimPrefix(opt.RefFullName, git.BranchPrefix) - if len(branch) == 0 || opt.PusherID <= 0 { - ctx.Error(404) - log.Trace("PushUpdate: branch or secret is empty, or pusher ID is not valid") - return - } - - err := models.PushUpdate(branch, opt) - if err != nil { - if models.IsErrUserNotExist(err) { - ctx.Error(404) - } else { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - } - return - } - ctx.Status(202) -} diff --git a/routers/private/repository.go b/routers/private/repository.go deleted file mode 100644 index 9f451bcf1dbb..000000000000 --- a/routers/private/repository.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "net/http" - - "code.gitea.io/gitea/models" - - macaron "gopkg.in/macaron.v1" -) - -// GetRepository return the default branch of a repository -func GetRepository(ctx *macaron.Context) { - repoID := ctx.ParamsInt64(":rid") - repository, err := models.GetRepositoryByID(repoID) - repository.MustOwnerName() - allowPulls := repository.AllowsPulls() - // put it back to nil because json unmarshal can't unmarshal it - repository.Units = nil - - if err != nil { - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": err.Error(), - }) - return - } - - if repository.IsFork { - repository.GetBaseRepo() - if err != nil { - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": err.Error(), - }) - return - } - repository.BaseRepo.MustOwnerName() - allowPulls = repository.BaseRepo.AllowsPulls() - // put it back to nil because json unmarshal can't unmarshal it - repository.BaseRepo.Units = nil - } - - ctx.JSON(http.StatusOK, struct { - Repository *models.Repository - AllowPullRequest bool - }{ - Repository: repository, - AllowPullRequest: allowPulls, - }) -} - -// GetActivePullRequest return an active pull request when it exists or an empty object -func GetActivePullRequest(ctx *macaron.Context) { - baseRepoID := ctx.QueryInt64("baseRepoID") - headRepoID := ctx.QueryInt64("headRepoID") - baseBranch := ctx.QueryTrim("baseBranch") - if len(baseBranch) == 0 { - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": "QueryTrim failed", - }) - return - } - - headBranch := ctx.QueryTrim("headBranch") - if len(headBranch) == 0 { - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": "QueryTrim failed", - }) - return - } - - pr, err := models.GetUnmergedPullRequest(headRepoID, baseRepoID, headBranch, baseBranch) - if err != nil && !models.IsErrPullRequestNotExist(err) { - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": err.Error(), - }) - return - } - - ctx.JSON(http.StatusOK, pr) -} diff --git a/routers/private/wiki.go b/routers/private/wiki.go deleted file mode 100644 index 33bcbaf17ea6..000000000000 --- a/routers/private/wiki.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package private - -import ( - "code.gitea.io/gitea/models" - - macaron "gopkg.in/macaron.v1" -) - -// InitWiki initilizes wiki via repo id -func InitWiki(ctx *macaron.Context) { - repoID := ctx.ParamsInt64("repoid") - - repo, err := models.GetRepositoryByID(repoID) - if err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - - err = repo.InitWiki() - if err != nil { - ctx.JSON(500, map[string]interface{}{ - "err": err.Error(), - }) - return - } - - ctx.Status(202) -} From 8ff661865cd8fe1cf3bc599467598d6d4925dc85 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 19 May 2019 20:52:54 +0100 Subject: [PATCH 04/25] finally remove the gitlogger --- cmd/serv.go | 14 ++++++++------ .../advanced/logging-documentation.en-us.md | 16 ---------------- modules/log/log.go | 19 +------------------ modules/pprof/pprof.go | 15 +++++++-------- routers/init.go | 2 -- routers/repo/http.go | 8 ++++---- 6 files changed, 20 insertions(+), 54 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 6ef12c6aa732..aa068d4cf693 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -12,7 +12,6 @@ import ( "net/url" "os" "os/exec" - "path/filepath" "strconv" "strings" "time" @@ -71,7 +70,6 @@ func setup(logPath string) { log.DelLogger("console") setting.NewContext() checkLFSVersion() - log.NewGitLogger(filepath.Join(setting.LogRootPath, logPath)) } func parseCmd(cmd string) (string, string) { @@ -98,11 +96,9 @@ func fail(userMessage, logMessage string, args ...interface{}) { if !setting.ProdMode { fmt.Fprintf(os.Stderr, logMessage+"\n", args...) } - log.GitLogger.Fatal(logMessage, args...) return } - log.GitLogger.Close() os.Exit(1) } @@ -170,10 +166,16 @@ func runServ(c *cli.Context) error { fail("Error while trying to create PPROF_DATA_PATH", "Error while trying to create PPROF_DATA_PATH: %v", err) } - stopCPUProfiler := pprof.DumpCPUProfileForUsername(setting.PprofDataPath, username) + stopCPUProfiler, err := pprof.DumpCPUProfileForUsername(setting.PprofDataPath, username) + if err != nil { + fail("Internal Server Error", "Unable to start CPU profile: %v", err) + } defer func() { stopCPUProfiler() - pprof.DumpMemProfileForUsername(setting.PprofDataPath, username) + err := pprof.DumpMemProfileForUsername(setting.PprofDataPath, username) + if err != nil { + fail("Internal Server Error", "Unable to dump Mem Profile: %v", err) + } }() } diff --git a/docs/content/doc/advanced/logging-documentation.en-us.md b/docs/content/doc/advanced/logging-documentation.en-us.md index df3578694346..790e750084ef 100644 --- a/docs/content/doc/advanced/logging-documentation.en-us.md +++ b/docs/content/doc/advanced/logging-documentation.en-us.md @@ -27,7 +27,6 @@ log groups: * The Router logger * The Access logger * The XORM logger -* A logger called the `GitLogger` which is used during hooks. There is also the go log logger. @@ -180,21 +179,6 @@ which will not be inherited from the `[log]` or relevant * `EXPRESSION` will default to `""` * `PREFIX` will default to `""` -### The Hook and Serv "GitLoggers" - -These are less well defined loggers. Essentially these should only be -used within Gitea's subsystems and cannot be configured at present. - -They will write log files in: - -* `%(ROOT_PATH)/hooks/pre-receive.log` -* `%(ROOT_PATH)/hooks/update.log` -* `%(ROOT_PATH)/hooks/post-receive.log` -* `%(ROOT_PATH)/serv.log` -* `%(ROOT_PATH)/http.log` - -In the future these logs may be rationalised. - ## Log outputs Gitea provides 4 possible log outputs: diff --git a/modules/log/log.go b/modules/log/log.go index d18996d48d31..8698e9eed3ae 100644 --- a/modules/log/log.go +++ b/modules/log/log.go @@ -5,9 +5,7 @@ package log import ( - "fmt" "os" - "path" "runtime" "strings" ) @@ -17,9 +15,7 @@ var ( DEFAULT = "default" // NamedLoggers map of named loggers NamedLoggers = make(map[string]*Logger) - // GitLogger logger for git - GitLogger *Logger - prefix string + prefix string ) // NewLogger create a logger for the default logger @@ -72,19 +68,6 @@ func GetLogger(name string) *Logger { return NamedLoggers[DEFAULT] } -// NewGitLogger create a logger for git -// FIXME: use same log level as other loggers. -func NewGitLogger(logPath string) { - path := path.Dir(logPath) - - if err := os.MkdirAll(path, os.ModePerm); err != nil { - Fatal("Failed to create dir %s: %v", path, err) - } - - GitLogger = newLogger("git", 0) - GitLogger.SetLogger("file", "file", fmt.Sprintf(`{"level":"TRACE","filename":"%s","rotate":true,"maxsize":%d,"daily":true,"maxdays":7,"compress":true,"compressionLevel":-1, "stacktraceLevel":"NONE"}`, logPath, 1<<28)) -} - // GetLevel returns the minimum logger level func GetLevel() Level { return NamedLoggers[DEFAULT].GetLevel() diff --git a/modules/pprof/pprof.go b/modules/pprof/pprof.go index e02c2d0f2aba..4b8f38ffac5e 100644 --- a/modules/pprof/pprof.go +++ b/modules/pprof/pprof.go @@ -9,34 +9,33 @@ import ( "io/ioutil" "runtime" "runtime/pprof" - - "code.gitea.io/gitea/modules/log" ) // DumpMemProfileForUsername dumps a memory profile at pprofDataPath as memprofile__ -func DumpMemProfileForUsername(pprofDataPath, username string) { +func DumpMemProfileForUsername(pprofDataPath, username string) error { f, err := ioutil.TempFile(pprofDataPath, fmt.Sprintf("memprofile_%s_", username)) if err != nil { - log.GitLogger.Fatal("Could not create memory profile: %v", err) + return err } defer f.Close() runtime.GC() // get up-to-date statistics if err := pprof.WriteHeapProfile(f); err != nil { - log.GitLogger.Fatal("Could not write memory profile: %v", err) + return err } + return nil } // DumpCPUProfileForUsername dumps a CPU profile at pprofDataPath as cpuprofile__ // it returns the stop function which stops, writes and closes the CPU profile file -func DumpCPUProfileForUsername(pprofDataPath, username string) func() { +func DumpCPUProfileForUsername(pprofDataPath, username string) (func(), error) { f, err := ioutil.TempFile(pprofDataPath, fmt.Sprintf("cpuprofile_%s_", username)) if err != nil { - log.GitLogger.Fatal("Could not create cpu profile: %v", err) + return nil, err } pprof.StartCPUProfile(f) return func() { pprof.StopCPUProfile() f.Close() - } + }, nil } diff --git a/routers/init.go b/routers/init.go index 88422cc6ede0..c6591efee7c6 100644 --- a/routers/init.go +++ b/routers/init.go @@ -5,7 +5,6 @@ package routers import ( - "path" "strings" "time" @@ -97,7 +96,6 @@ func GlobalInit() { models.InitSyncMirrors() models.InitDeliverHooks() models.InitTestPullRequests() - log.NewGitLogger(path.Join(setting.LogRootPath, "http.log")) } if models.EnableSQLite3 { log.Info("SQLite3 Supported") diff --git a/routers/repo/http.go b/routers/repo/http.go index fccecfb71d2b..214e2f341133 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -351,7 +351,7 @@ func gitCommand(dir string, args ...string) []byte { cmd.Dir = dir out, err := cmd.Output() if err != nil { - log.GitLogger.Error(fmt.Sprintf("%v - %s", err, out)) + log.Error("%v - %s", err, out) } return out } @@ -409,7 +409,7 @@ func serviceRPC(h serviceHandler, service string) { if h.r.Header.Get("Content-Encoding") == "gzip" { reqBody, err = gzip.NewReader(reqBody) if err != nil { - log.GitLogger.Error("Fail to create gzip reader: %v", err) + log.Error("Fail to create gzip reader: %v", err) h.w.WriteHeader(http.StatusInternalServerError) return } @@ -428,7 +428,7 @@ func serviceRPC(h serviceHandler, service string) { cmd.Stdin = reqBody cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - log.GitLogger.Error("Fail to serve RPC(%s): %v - %v", service, err, stderr) + log.Error("Fail to serve RPC(%s): %v - %v", service, err, stderr) return } } @@ -541,7 +541,7 @@ func HTTPBackend(ctx *context.Context, cfg *serviceConfig) http.HandlerFunc { file := strings.Replace(r.URL.Path, m[1]+"/", "", 1) dir, err := getGitRepoPath(m[1]) if err != nil { - log.GitLogger.Error(err.Error()) + log.Error(err.Error()) ctx.NotFound("HTTPBackend", err) return } From 9cc4043ee36b34d299d46fee0ffe7ce63f80c60a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 19 May 2019 20:59:27 +0100 Subject: [PATCH 05/25] Disallow push on archived repositories --- routers/private/serv.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/routers/private/serv.go b/routers/private/serv.go index 3b8c6cab4060..ef9504fe8af7 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -210,6 +210,16 @@ func ServCommand(ctx *macaron.Context) { results.UserName = user.Name } + // Don't allow pushing if the repo is archived + if mode > models.AccessModeRead && repo.IsArchived { + ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ + "results": results, + "type": "ErrRepoIsArchived", + "err": fmt.Sprintf("Repo: %s/%s is archived.", results.OwnerName, results.RepoName), + }) + return + } + // Permissions checking: if mode > models.AccessModeRead || repo.IsPrivate || setting.Service.RequireSignInView { if key.Type == models.KeyTypeDeploy { @@ -217,7 +227,7 @@ func ServCommand(ctx *macaron.Context) { ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ "results": results, "type": "ErrUnauthorized", - "err": fmt.Sprintf("Deploy Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, ownerName, repoName), + "err": fmt.Sprintf("Deploy Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, results.OwnerName, results.RepoName), }) return } @@ -228,7 +238,7 @@ func ServCommand(ctx *macaron.Context) { ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ "results": results, "type": "InternalServerError", - "err": fmt.Sprintf("Unable to get permissions for user %d:%s with key %d in %s/%s Error: %v", user.ID, user.Name, key.ID, ownerName, repoName, err), + "err": fmt.Sprintf("Unable to get permissions for user %d:%s with key %d in %s/%s Error: %v", user.ID, user.Name, key.ID, results.OwnerName, results.RepoName, err), }) return } From f26ac93ab5f9b6003231e89c2d689148231b50a8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 19 May 2019 21:21:51 +0100 Subject: [PATCH 06/25] fix lint error --- modules/pprof/pprof.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/pprof/pprof.go b/modules/pprof/pprof.go index 4b8f38ffac5e..b63904e71362 100644 --- a/modules/pprof/pprof.go +++ b/modules/pprof/pprof.go @@ -19,10 +19,7 @@ func DumpMemProfileForUsername(pprofDataPath, username string) error { } defer f.Close() runtime.GC() // get up-to-date statistics - if err := pprof.WriteHeapProfile(f); err != nil { - return err - } - return nil + return pprof.WriteHeapProfile(f) } // DumpCPUProfileForUsername dumps a CPU profile at pprofDataPath as cpuprofile__ From 57c9f078afec58cb70d1d67ac5742f95889e3078 Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 20 May 2019 03:30:54 +0100 Subject: [PATCH 07/25] Update modules/private/key.go --- modules/private/key.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/private/key.go b/modules/private/key.go index f81dd24d9084..ebc28eb87139 100644 --- a/modules/private/key.go +++ b/modules/private/key.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. From bdce608010711321655a0c896df0783af2a54a1c Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 20 May 2019 03:31:26 +0100 Subject: [PATCH 08/25] Update routers/private/hook.go --- routers/private/hook.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/private/hook.go b/routers/private/hook.go index a2a12f54c2a4..6f39c0593a67 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" + macaron "gopkg.in/macaron.v1" ) From 02cba7f88413cb5202cea55a16c8c92efe8db428 Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 20 May 2019 03:32:05 +0100 Subject: [PATCH 09/25] Update routers/private/hook.go --- routers/private/hook.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/private/hook.go b/routers/private/hook.go index 6f39c0593a67..5460aa0ec200 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -140,6 +140,7 @@ func HookPostReceive(ctx *macaron.Context) { ctx.JSON(http.StatusOK, map[string]interface{}{ "message": false, }) + return } baseRepo := repo From ee7f7925f3535823be2a17b064417924e6c9253d Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 20 May 2019 03:32:36 +0100 Subject: [PATCH 10/25] Update routers/private/hook.go --- routers/private/hook.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/private/hook.go b/routers/private/hook.go index 5460aa0ec200..564829fb5cf3 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -159,6 +159,7 @@ func HookPostReceive(ctx *macaron.Context) { ctx.JSON(http.StatusOK, map[string]interface{}{ "message": false, }) + return } pr, err := models.GetUnmergedPullRequest(repo.ID, baseRepo.ID, branch, baseRepo.DefaultBranch) From f6659b319e716013709ff8cfeb3c1d50929bbecd Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 20 May 2019 03:33:12 +0100 Subject: [PATCH 11/25] Updated routers/private/serv.go --- routers/private/serv.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/private/serv.go b/routers/private/serv.go index ef9504fe8af7..68e4361e56d8 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" + macaron "gopkg.in/macaron.v1" ) From f6d76238de45c63c18b437820c0d7ee2ed9afdf3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 20 May 2019 09:38:11 +0100 Subject: [PATCH 12/25] Fix LFS Locks over SSH --- integrations/git_test.go | 97 +++++++++++++++++++++------------ modules/lfs/locks.go | 115 ++++++++++++++++++++++++++++----------- routers/routes/routes.go | 2 +- 3 files changed, 146 insertions(+), 68 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index ebbf04f9d084..b052cb3e090f 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -65,6 +65,9 @@ func testGit(t *testing.T, u *url.URL) { little = commitAndPush(t, littleSize, dstPath) }) t.Run("Big", func(t *testing.T) { + if testing.Short() { + return + } PrintCurrentTest(t) big = commitAndPush(t, bigSize, dstPath) }) @@ -85,10 +88,15 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Little", func(t *testing.T) { PrintCurrentTest(t) littleLFS = commitAndPush(t, littleSize, dstPath) + lockFileTest(t, littleLFS, dstPath) }) t.Run("Big", func(t *testing.T) { + if testing.Short() { + return + } PrintCurrentTest(t) bigLFS = commitAndPush(t, bigSize, dstPath) + lockFileTest(t, bigLFS, dstPath) }) }) t.Run("Locks", func(t *testing.T) { @@ -105,19 +113,21 @@ func testGit(t *testing.T, u *url.URL) { resp := session.MakeRequest(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", big)) - nilResp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, nilResp.Length) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", littleLFS)) resp = session.MakeRequest(t, req, http.StatusOK) assert.NotEqual(t, littleSize, resp.Body.Len()) assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, bigSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", big)) + nilResp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, bigSize, nilResp.Length) + + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", bigLFS)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.NotEqual(t, bigSize, resp.Body.Len()) + assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + } }) t.Run("Media", func(t *testing.T) { @@ -129,17 +139,19 @@ func testGit(t *testing.T, u *url.URL) { resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Length) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", big)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Length) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", littleLFS)) resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Length) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", bigLFS)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Length) + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", big)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Length) + + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", bigLFS)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Length) + } }) }) @@ -177,6 +189,9 @@ func testGit(t *testing.T, u *url.URL) { little = commitAndPush(t, littleSize, dstPath) }) t.Run("Big", func(t *testing.T) { + if testing.Short() { + return + } PrintCurrentTest(t) big = commitAndPush(t, bigSize, dstPath) }) @@ -197,10 +212,17 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Little", func(t *testing.T) { PrintCurrentTest(t) littleLFS = commitAndPush(t, littleSize, dstPath) + lockFileTest(t, littleLFS, dstPath) + }) t.Run("Big", func(t *testing.T) { + if testing.Short() { + return + } PrintCurrentTest(t) bigLFS = commitAndPush(t, bigSize, dstPath) + lockFileTest(t, bigLFS, dstPath) + }) }) t.Run("Locks", func(t *testing.T) { @@ -217,20 +239,21 @@ func testGit(t *testing.T, u *url.URL) { resp := session.MakeRequest(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", big)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", littleLFS)) resp = session.MakeRequest(t, req, http.StatusOK) assert.NotEqual(t, littleSize, resp.Body.Len()) assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, bigSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", big)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Body.Len()) + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", bigLFS)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.NotEqual(t, bigSize, resp.Body.Len()) + assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + } }) t.Run("Media", func(t *testing.T) { PrintCurrentTest(t) @@ -241,17 +264,19 @@ func testGit(t *testing.T, u *url.URL) { resp := session.MakeRequest(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", big)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", littleLFS)) resp = session.MakeRequest(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", big)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Body.Len()) + + req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", bigLFS)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Body.Len()) + } }) }) @@ -268,15 +293,17 @@ func ensureAnonymousClone(t *testing.T, u *url.URL) { } func lockTest(t *testing.T, remote, repoPath string) { - _, err := git.NewCommand("remote").AddArguments("set-url", "origin", remote).RunInDir(repoPath) //TODO add test ssh git-lfs-creds - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(repoPath) + lockFileTest(t, "README.md", repoPath) +} + +func lockFileTest(t *testing.T, filename, repoPath string) { + _, err := git.NewCommand("lfs").AddArguments("locks").RunInDir(repoPath) assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("lock", "README.md").RunInDir(repoPath) + _, err = git.NewCommand("lfs").AddArguments("lock", filename).RunInDir(repoPath) assert.NoError(t, err) _, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(repoPath) assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("unlock", "README.md").RunInDir(repoPath) + _, err = git.NewCommand("lfs").AddArguments("unlock", filename).RunInDir(repoPath) assert.NoError(t, err) } diff --git a/modules/lfs/locks.go b/modules/lfs/locks.go index 525a93645f78..b1ca2f094a25 100644 --- a/modules/lfs/locks.go +++ b/modules/lfs/locks.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" ) @@ -44,7 +45,7 @@ func checkIsValidRequest(ctx *context.Context, post bool) bool { return true } -func handleLockListOut(ctx *context.Context, lock *models.LFSLock, err error) { +func handleLockListOut(ctx *context.Context, repo *models.Repository, lock *models.LFSLock, err error) { if err != nil { if models.IsErrLFSLockNotExist(err) { ctx.JSON(200, api.LFSLockList{ @@ -57,7 +58,7 @@ func handleLockListOut(ctx *context.Context, lock *models.LFSLock, err error) { }) return } - if ctx.Repo.Repository.ID != lock.RepoID { + if repo.ID != lock.RepoID { ctx.JSON(200, api.LFSLockList{ Locks: []*api.LFSLock{}, }) @@ -75,17 +76,21 @@ func GetListLockHandler(ctx *context.Context) { } ctx.Resp.Header().Set("Content-Type", metaMediaType) - err := models.CheckLFSAccessForRepo(ctx.User, ctx.Repo.Repository, models.AccessModeRead) + rv := unpack(ctx) + + repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo) if err != nil { - if models.IsErrLFSUnauthorizedAction(err) { - ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") - ctx.JSON(401, api.LFSLockError{ - Message: "You must have pull access to list locks : " + err.Error(), - }) - return - } - ctx.JSON(500, api.LFSLockError{ - Message: "unable to list lock : " + err.Error(), + log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err) + writeStatus(ctx, 404) + return + } + repository.MustOwner() + + authenticated := authenticate(ctx, repository, rv.Authorization, false) + if !authenticated { + ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") + ctx.JSON(401, api.LFSLockError{ + Message: "You must have pull access to list locks", }) return } @@ -100,19 +105,19 @@ func GetListLockHandler(ctx *context.Context) { return } lock, err := models.GetLFSLockByID(int64(v)) - handleLockListOut(ctx, lock, err) + handleLockListOut(ctx, repository, lock, err) return } path := ctx.Query("path") if path != "" { //Case where we request a specific id - lock, err := models.GetLFSLock(ctx.Repo.Repository, path) - handleLockListOut(ctx, lock, err) + lock, err := models.GetLFSLock(repository, path) + handleLockListOut(ctx, repository, lock, err) return } //If no query params path or id - lockList, err := models.GetLFSLockByRepoID(ctx.Repo.Repository.ID) + lockList, err := models.GetLFSLockByRepoID(repository.ID) if err != nil { ctx.JSON(500, api.LFSLockError{ Message: "unable to list locks : " + err.Error(), @@ -135,16 +140,36 @@ func PostLockHandler(ctx *context.Context) { } ctx.Resp.Header().Set("Content-Type", metaMediaType) + userName := ctx.Params("username") + repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") + authorization := ctx.Req.Header.Get("Authorization") + + repository, err := models.GetRepositoryByOwnerAndName(userName, repoName) + if err != nil { + log.Debug("Could not find repository: %s/%s - %s", userName, repoName, err) + writeStatus(ctx, 404) + return + } + repository.MustOwner() + + authenticated := authenticate(ctx, repository, authorization, true) + if !authenticated { + ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") + ctx.JSON(401, api.LFSLockError{ + Message: "You must have push access to create locks", + }) + return + } + var req api.LFSLockRequest dec := json.NewDecoder(ctx.Req.Body().ReadCloser()) - err := dec.Decode(&req) - if err != nil { + if err := dec.Decode(&req); err != nil { writeStatus(ctx, 400) return } lock, err := models.CreateLFSLock(&models.LFSLock{ - Repo: ctx.Repo.Repository, + Repo: repository, Path: req.Path, Owner: ctx.User, }) @@ -178,23 +203,29 @@ func VerifyLockHandler(ctx *context.Context) { } ctx.Resp.Header().Set("Content-Type", metaMediaType) - err := models.CheckLFSAccessForRepo(ctx.User, ctx.Repo.Repository, models.AccessModeWrite) + userName := ctx.Params("username") + repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") + authorization := ctx.Req.Header.Get("Authorization") + + repository, err := models.GetRepositoryByOwnerAndName(userName, repoName) if err != nil { - if models.IsErrLFSUnauthorizedAction(err) { - ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") - ctx.JSON(401, api.LFSLockError{ - Message: "You must have push access to verify locks : " + err.Error(), - }) - return - } - ctx.JSON(500, api.LFSLockError{ - Message: "unable to verify lock : " + err.Error(), + log.Debug("Could not find repository: %s/%s - %s", userName, repoName, err) + writeStatus(ctx, 404) + return + } + repository.MustOwner() + + authenticated := authenticate(ctx, repository, authorization, true) + if !authenticated { + ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") + ctx.JSON(401, api.LFSLockError{ + Message: "You must have push access to verify locks", }) return } //TODO handle body json cursor and limit - lockList, err := models.GetLFSLockByRepoID(ctx.Repo.Repository.ID) + lockList, err := models.GetLFSLockByRepoID(repository.ID) if err != nil { ctx.JSON(500, api.LFSLockError{ Message: "unable to list locks : " + err.Error(), @@ -223,10 +254,30 @@ func UnLockHandler(ctx *context.Context) { } ctx.Resp.Header().Set("Content-Type", metaMediaType) + userName := ctx.Params("username") + repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") + authorization := ctx.Req.Header.Get("Authorization") + + repository, err := models.GetRepositoryByOwnerAndName(userName, repoName) + if err != nil { + log.Debug("Could not find repository: %s/%s - %s", userName, repoName, err) + writeStatus(ctx, 404) + return + } + repository.MustOwner() + + authenticated := authenticate(ctx, repository, authorization, true) + if !authenticated { + ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") + ctx.JSON(401, api.LFSLockError{ + Message: "You must have push access to delete locks", + }) + return + } + var req api.LFSLockDeleteRequest dec := json.NewDecoder(ctx.Req.Body().ReadCloser()) - err := dec.Decode(&req) - if err != nil { + if err := dec.Decode(&req); err != nil { writeStatus(ctx, 400) return } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 5a5fc518b927..d19823714b06 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -923,7 +923,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/", lfs.PostLockHandler) m.Post("/verify", lfs.VerifyLockHandler) m.Post("/:lid/unlock", lfs.UnLockHandler) - }, context.RepoAssignment()) + }) m.Any("/*", func(ctx *context.Context) { ctx.NotFound("", nil) }) From cb9f80a18e724d3a186ae92ac2e00fac70d26049 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 20 May 2019 13:31:54 +0100 Subject: [PATCH 13/25] rev-list needs to be run by the hook process --- cmd/hook.go | 7 +++++++ modules/private/hook.go | 8 ++++++-- routers/private/hook.go | 17 +++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index 5b48e20f5bf8..e886e8a250c0 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -150,12 +150,19 @@ func runHookPostReceive(c *cli.Context) error { newCommitID := string(fields[1]) refFullName := string(fields[2]) + output, errErr := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).Run() + err := "" + if errErr != nil { + err = errErr.Error() + } res, err := private.HookPostReceive(repoUser, repoName, private.HookOptions{ OldCommitID: oldCommitID, NewCommitID: newCommitID, RefFullName: refFullName, UserID: pusherID, UserName: pusherName, + Output: output, + Err: err, }) if res == nil { diff --git a/modules/private/hook.go b/modules/private/hook.go index 69729b8c32d7..6bf7145f9bdc 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -20,17 +20,21 @@ type HookOptions struct { RefFullName string UserID int64 UserName string + Output string + Err string } // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d", + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&output=%s&err=%s", url.PathEscape(ownerName), url.PathEscape(repoName), url.QueryEscape(opts.OldCommitID), url.QueryEscape(opts.NewCommitID), url.QueryEscape(opts.RefFullName), - opts.UserID) + opts.UserID, + url.QueryEscape(opts.Output), + url.QueryEscape(opts.Err)) resp, err := newInternalRequest(reqURL, "GET").Response() if err != nil { diff --git a/routers/private/hook.go b/routers/private/hook.go index a2a12f54c2a4..a138ba049a8f 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -21,10 +21,12 @@ import ( func HookPreReceive(ctx *macaron.Context) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") - oldCommitID := ctx.Query("old") - newCommitID := ctx.Query("new") - refFullName := ctx.Query("ref") + oldCommitID := ctx.QueryTrim("old") + newCommitID := ctx.QueryTrim("new") + refFullName := ctx.QueryTrim("ref") userID := ctx.QueryInt64("userID") + revListOutput := ctx.Query("output") + revListErr := ctx.Query("err") branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) @@ -56,14 +58,13 @@ func HookPreReceive(ctx *macaron.Context) { // detect force push if git.EmptySHA != oldCommitID { - output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDir(repo.RepoPath()) - if err != nil { - log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + if revListErr != "" { + log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, revListErr) ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Fail to detect force push: %v", err), + "err": fmt.Sprintf("Fail to detect force push: %v", revListErr), }) return - } else if len(output) > 0 { + } else if len(revListOutput) > 0 { log.Warn("Forbidden: Branch: %s in %-v is protected from force push", branchName, repo) ctx.JSON(http.StatusForbidden, map[string]interface{}{ "err": fmt.Sprintf("branch %s is protected from force push", branchName), From 1b58ab4df3a1f5b0dedf25e823b48d971a9a4462 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 20 May 2019 14:01:45 +0100 Subject: [PATCH 14/25] fixup --- cmd/hook.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index e886e8a250c0..f8aa32f33c56 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -86,6 +86,11 @@ func runHookPreReceive(c *cli.Context) error { newCommitID := string(fields[1]) refFullName := string(fields[2]) + output, errErr := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).Run() + err := "" + if errErr != nil { + err = errErr.Error() + } // If the ref is a branch, check if it's protected if strings.HasPrefix(refFullName, git.BranchPrefix) { statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{ @@ -93,6 +98,8 @@ func runHookPreReceive(c *cli.Context) error { NewCommitID: newCommitID, RefFullName: refFullName, UserID: userID, + Output: output, + Err: err, }) switch statusCode { case http.StatusInternalServerError: @@ -150,19 +157,12 @@ func runHookPostReceive(c *cli.Context) error { newCommitID := string(fields[1]) refFullName := string(fields[2]) - output, errErr := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).Run() - err := "" - if errErr != nil { - err = errErr.Error() - } res, err := private.HookPostReceive(repoUser, repoName, private.HookOptions{ OldCommitID: oldCommitID, NewCommitID: newCommitID, RefFullName: refFullName, UserID: pusherID, UserName: pusherName, - Output: output, - Err: err, }) if res == nil { From 2fe133fbe715f5d2eda3407c88d4f128aa9bea35 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 20 May 2019 14:28:00 +0100 Subject: [PATCH 15/25] Improve git test --- .../api_helper_for_declarative_test.go | 42 ++++++++++ .../git_helper_for_declarative_test.go | 36 +++++++- integrations/git_test.go | 82 +++++++++++++++++-- 3 files changed, 150 insertions(+), 10 deletions(-) diff --git a/integrations/api_helper_for_declarative_test.go b/integrations/api_helper_for_declarative_test.go index 943981ead2f0..85f0ab621f84 100644 --- a/integrations/api_helper_for_declarative_test.go +++ b/integrations/api_helper_for_declarative_test.go @@ -5,11 +5,14 @@ package integrations import ( + "encoding/json" "fmt" "io/ioutil" "net/http" "testing" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" api "code.gitea.io/gitea/modules/structs" "github.com/stretchr/testify/assert" ) @@ -150,3 +153,42 @@ func doAPICreateDeployKey(ctx APITestContext, keyname, keyFile string, readOnly ctx.Session.MakeRequest(t, req, http.StatusCreated) } } + +func doAPICreatePullRequest(ctx APITestContext, owner, repo, baseBranch, headBranch string) func(*testing.T) (api.PullRequest, error) { + return func(t *testing.T) (api.PullRequest, error) { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/pulls?token=%s", + owner, repo, ctx.Token) + req := NewRequestWithJSON(t, http.MethodPost, urlStr, &api.CreatePullRequestOption{ + Head: headBranch, + Base: baseBranch, + Title: fmt.Sprintf("create a pr from %s to %s", headBranch, baseBranch), + }) + + expected := 201 + if ctx.ExpectedCode != 0 { + expected = ctx.ExpectedCode + } + resp := ctx.Session.MakeRequest(t, req, expected) + decoder := json.NewDecoder(resp.Body) + pr := api.PullRequest{} + err := decoder.Decode(&pr) + return pr, err + } +} + +func doAPIMergePullRequest(ctx APITestContext, owner, repo string, index int64) func(*testing.T) { + return func(t *testing.T) { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/merge?token=%s", + owner, repo, index, ctx.Token) + req := NewRequestWithJSON(t, http.MethodPost, urlStr, &auth.MergePullRequestForm{ + MergeMessageField: "doAPIMergePullRequest Merge", + Do: string(models.MergeStyleMerge), + }) + + if ctx.ExpectedCode != 0 { + ctx.Session.MakeRequest(t, req, ctx.ExpectedCode) + return + } + ctx.Session.MakeRequest(t, req, 200) + } +} diff --git a/integrations/git_helper_for_declarative_test.go b/integrations/git_helper_for_declarative_test.go index b4fead66253b..235f4b4a9b74 100644 --- a/integrations/git_helper_for_declarative_test.go +++ b/integrations/git_helper_for_declarative_test.go @@ -112,16 +112,44 @@ func doGitAddRemote(dstPath, remoteName string, u *url.URL) func(*testing.T) { } } -func doGitPushTestRepository(dstPath, remoteName, branch string) func(*testing.T) { +func doGitPushTestRepository(dstPath string, args ...string) func(*testing.T) { return func(t *testing.T) { - _, err := git.NewCommand("push", "-u", remoteName, branch).RunInDir(dstPath) + _, err := git.NewCommand(append([]string{"push", "-u"}, args...)...).RunInDir(dstPath) assert.NoError(t, err) } } -func doGitPushTestRepositoryFail(dstPath, remoteName, branch string) func(*testing.T) { +func doGitPushTestRepositoryFail(dstPath string, args ...string) func(*testing.T) { return func(t *testing.T) { - _, err := git.NewCommand("push", "-u", remoteName, branch).RunInDir(dstPath) + _, err := git.NewCommand(append([]string{"push"}, args...)...).RunInDir(dstPath) assert.Error(t, err) } } + +func doGitCreateBranch(dstPath, branch string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand("checkout", "-b", branch).RunInDir(dstPath) + assert.NoError(t, err) + } +} + +func doGitCheckoutBranch(dstPath string, args ...string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand(append([]string{"checkout"}, args...)...).RunInDir(dstPath) + assert.NoError(t, err) + } +} + +func doGitMerge(dstPath string, args ...string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand(append([]string{"merge"}, args...)...).RunInDir(dstPath) + assert.NoError(t, err) + } +} + +func doGitPull(dstPath string, args ...string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand(append([]string{"pull"}, args...)...).RunInDir(dstPath) + assert.NoError(t, err) + } +} diff --git a/integrations/git_test.go b/integrations/git_test.go index b052cb3e090f..15ffe33468a5 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -13,11 +13,13 @@ import ( "os" "path" "path/filepath" + "strconv" "testing" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + api "code.gitea.io/gitea/modules/structs" "github.com/stretchr/testify/assert" ) @@ -153,7 +155,7 @@ func testGit(t *testing.T, u *url.URL) { assert.Equal(t, bigSize, resp.Length) } }) - + t.Run("BranchProtectMerge", doBranchProtectPRMerge(httpContext.Username, httpContext.Reponame, dstPath)) }) t.Run("SSH", func(t *testing.T) { PrintCurrentTest(t) @@ -278,7 +280,7 @@ func testGit(t *testing.T, u *url.URL) { assert.Equal(t, bigSize, resp.Body.Len()) } }) - + t.Run("BranchProtectMerge", doBranchProtectPRMerge(sshContext.Username, sshContext.Reponame, dstPath)) }) }) @@ -308,21 +310,21 @@ func lockFileTest(t *testing.T, filename, repoPath string) { } func commitAndPush(t *testing.T, size int, repoPath string) string { - name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two") + name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", "data-file-") assert.NoError(t, err) - _, err = git.NewCommand("push").RunInDir(repoPath) //Push + _, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push assert.NoError(t, err) return name } -func generateCommitWithNewData(size int, repoPath, email, fullName string) (string, error) { +func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) { //Generate random file data := make([]byte, size) _, err := rand.Read(data) if err != nil { return "", err } - tmpFile, err := ioutil.TempFile(repoPath, "data-file-") + tmpFile, err := ioutil.TempFile(repoPath, prefix) if err != nil { return "", err } @@ -352,3 +354,71 @@ func generateCommitWithNewData(size int, repoPath, email, fullName string) (stri }) return filepath.Base(tmpFile.Name()), err } + +func doBranchProtectPRMerge(username, reponame, dstPath string) func(t *testing.T) { + return func(t *testing.T) { + PrintCurrentTest(t) + t.Run("CreateBranchProtected", doGitCreateBranch(dstPath, "protected")) + t.Run("PushProtectedBranch", doGitPushTestRepository(dstPath, "origin", "protected")) + + ctx := NewAPITestContext(t, username, reponame) + t.Run("ProtectProtectedBranchNoWhitelist", doProtectBranch(ctx, "protected", "")) + t.Run("GenerateCommit", func(t *testing.T) { + _, err := generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") + assert.NoError(t, err) + }) + t.Run("FailToPushToProtectedBranch", doGitPushTestRepositoryFail(dstPath, "origin", "protected")) + t.Run("PushToUnprotectedBranch", doGitPushTestRepository(dstPath, "origin", "protected:unprotected")) + var pr api.PullRequest + var err error + t.Run("CreatePullRequest", func(t *testing.T) { + pr, err = doAPICreatePullRequest(ctx, username, reponame, "protected", "unprotected")(t) + assert.NoError(t, err) + }) + t.Run("MergePR", doAPIMergePullRequest(ctx, username, reponame, pr.Index)) + t.Run("PullProtected", doGitPull(dstPath, "origin", "protected")) + t.Run("ProtectProtectedBranchWhitelist", doProtectBranch(ctx, "protected", username)) + + t.Run("CheckoutMaster", doGitCheckoutBranch(dstPath, "master")) + t.Run("CreateBranchForced", doGitCreateBranch(dstPath, "toforce")) + t.Run("GenerateCommit", func(t *testing.T) { + _, err := generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") + assert.NoError(t, err) + }) + t.Run("FailToForcePushToProtectedBranch", doGitPushTestRepositoryFail(dstPath, "-f", "origin", "toforce:protected")) + t.Run("MergeProtectedToToforce", doGitMerge(dstPath, "protected")) + t.Run("PushToProtectedBranch", doGitPushTestRepository(dstPath, "origin", "toforce:protected")) + t.Run("CheckoutMasterAgain", doGitCheckoutBranch(dstPath, "master")) + } +} + +func doProtectBranch(ctx APITestContext, branch string, userToWhitelist string) func(t *testing.T) { + // We are going to just use the owner to set the protection. + return func(t *testing.T) { + csrf := GetCSRF(t, ctx.Session, fmt.Sprintf("/%s/%s/settings/branches", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame))) + + if userToWhitelist == "" { + // Change branch to protected + req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings/branches/%s", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame), url.PathEscape(branch)), map[string]string{ + "_csrf": csrf, + "protected": "on", + }) + ctx.Session.MakeRequest(t, req, http.StatusFound) + } else { + user, err := models.GetUserByName(userToWhitelist) + assert.NoError(t, err) + // Change branch to protected + req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings/branches/%s", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame), url.PathEscape(branch)), map[string]string{ + "_csrf": csrf, + "protected": "on", + "enable_whitelist": "on", + "whitelist_users": strconv.FormatInt(user.ID, 10), + }) + ctx.Session.MakeRequest(t, req, http.StatusFound) + } + // Check if master branch has been locked successfully + flashCookie := ctx.Session.GetCookie("macaron_flash") + assert.NotNil(t, flashCookie) + assert.EqualValues(t, "success%3DBranch%2Bprotection%2Bfor%2Bbranch%2B%2527"+url.QueryEscape(branch)+"%2527%2Bhas%2Bbeen%2Bupdated.", flashCookie.Value) + } +} From b2d10b9a0346de773078dbd5e17cc1614d28150d Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 08:42:46 +0100 Subject: [PATCH 16/25] Ensure that the lfs files are created with a different prefix --- integrations/git_test.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index 15ffe33468a5..c3fbff06e742 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -62,16 +62,17 @@ func testGit(t *testing.T, u *url.URL) { t.Run("PushCommit", func(t *testing.T) { PrintCurrentTest(t) + prefix := "data-file-" t.Run("Little", func(t *testing.T) { PrintCurrentTest(t) - little = commitAndPush(t, littleSize, dstPath) + little = commitAndPush(t, littleSize, dstPath, prefix) }) t.Run("Big", func(t *testing.T) { if testing.Short() { return } PrintCurrentTest(t) - big = commitAndPush(t, bigSize, dstPath) + big = commitAndPush(t, bigSize, dstPath, prefix) }) }) }) @@ -80,16 +81,18 @@ func testGit(t *testing.T, u *url.URL) { t.Run("PushCommit", func(t *testing.T) { PrintCurrentTest(t) //Setup git LFS + prefix := "lfs-data-file-" + _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath) + _, err = git.NewCommand("lfs").AddArguments("track", prefix+"*").RunInDir(dstPath) assert.NoError(t, err) err = git.AddChanges(dstPath, false, ".gitattributes") assert.NoError(t, err) t.Run("Little", func(t *testing.T) { PrintCurrentTest(t) - littleLFS = commitAndPush(t, littleSize, dstPath) + littleLFS = commitAndPush(t, littleSize, dstPath, prefix) lockFileTest(t, littleLFS, dstPath) }) t.Run("Big", func(t *testing.T) { @@ -97,7 +100,7 @@ func testGit(t *testing.T, u *url.URL) { return } PrintCurrentTest(t) - bigLFS = commitAndPush(t, bigSize, dstPath) + bigLFS = commitAndPush(t, bigSize, dstPath, prefix) lockFileTest(t, bigLFS, dstPath) }) }) @@ -186,34 +189,37 @@ func testGit(t *testing.T, u *url.URL) { //time.Sleep(5 * time.Minute) t.Run("PushCommit", func(t *testing.T) { PrintCurrentTest(t) + prefix := "data-file-" t.Run("Little", func(t *testing.T) { PrintCurrentTest(t) - little = commitAndPush(t, littleSize, dstPath) + little = commitAndPush(t, littleSize, dstPath, prefix) }) t.Run("Big", func(t *testing.T) { if testing.Short() { return } PrintCurrentTest(t) - big = commitAndPush(t, bigSize, dstPath) + big = commitAndPush(t, bigSize, dstPath, prefix) }) }) }) t.Run("LFS", func(t *testing.T) { PrintCurrentTest(t) + t.Run("PushCommit", func(t *testing.T) { PrintCurrentTest(t) //Setup git LFS + prefix := "lfs-data-file-" _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath) + _, err = git.NewCommand("lfs").AddArguments("track", prefix+"*").RunInDir(dstPath) assert.NoError(t, err) err = git.AddChanges(dstPath, false, ".gitattributes") assert.NoError(t, err) t.Run("Little", func(t *testing.T) { PrintCurrentTest(t) - littleLFS = commitAndPush(t, littleSize, dstPath) + littleLFS = commitAndPush(t, littleSize, dstPath, prefix) lockFileTest(t, littleLFS, dstPath) }) @@ -222,7 +228,7 @@ func testGit(t *testing.T, u *url.URL) { return } PrintCurrentTest(t) - bigLFS = commitAndPush(t, bigSize, dstPath) + bigLFS = commitAndPush(t, bigSize, dstPath, prefix) lockFileTest(t, bigLFS, dstPath) }) @@ -309,8 +315,8 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } -func commitAndPush(t *testing.T, size int, repoPath string) string { - name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", "data-file-") +func commitAndPush(t *testing.T, size int, repoPath, prefix string) string { + name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) _, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push assert.NoError(t, err) From 4a967830cb4cc3b2265492bdba390f89e64ff9f1 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 09:37:39 +0100 Subject: [PATCH 17/25] Reduce the replication in git_test.go --- integrations/git_test.go | 346 +++++++++++++++------------------------ 1 file changed, 129 insertions(+), 217 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index c3fbff06e742..c37ac4b12548 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -45,119 +45,21 @@ func testGit(t *testing.T, u *url.URL) { httpContext.Reponame = "repo-tmp-17" dstPath, err := ioutil.TempDir("", httpContext.Reponame) - var little, big, littleLFS, bigLFS string - assert.NoError(t, err) defer os.RemoveAll(dstPath) - t.Run("Standard", func(t *testing.T) { - PrintCurrentTest(t) - ensureAnonymousClone(t, u) - - t.Run("CreateRepo", doAPICreateRepository(httpContext, false)) - - u.Path = httpContext.GitPath() - u.User = url.UserPassword(username, userPassword) - - t.Run("Clone", doGitClone(dstPath, u)) - - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - prefix := "data-file-" - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - little = commitAndPush(t, littleSize, dstPath, prefix) - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - return - } - PrintCurrentTest(t) - big = commitAndPush(t, bigSize, dstPath, prefix) - }) - }) - }) - t.Run("LFS", func(t *testing.T) { - PrintCurrentTest(t) - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - //Setup git LFS - prefix := "lfs-data-file-" - - _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("track", prefix+"*").RunInDir(dstPath) - assert.NoError(t, err) - err = git.AddChanges(dstPath, false, ".gitattributes") - assert.NoError(t, err) - - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - littleLFS = commitAndPush(t, littleSize, dstPath, prefix) - lockFileTest(t, littleLFS, dstPath) - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - return - } - PrintCurrentTest(t) - bigLFS = commitAndPush(t, bigSize, dstPath, prefix) - lockFileTest(t, bigLFS, dstPath) - }) - }) - t.Run("Locks", func(t *testing.T) { - PrintCurrentTest(t) - lockTest(t, u.String(), dstPath) - }) - }) - t.Run("Raw", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") - - // Request raw paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", little)) - resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", littleLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, littleSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", big)) - nilResp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, nilResp.Length) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, bigSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - } + t.Run("CreateRepo", doAPICreateRepository(httpContext, false)) - }) - t.Run("Media", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") + u.Path = httpContext.GitPath() + u.User = url.UserPassword(username, userPassword) - // Request media paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", little)) - resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Length) + t.Run("Clone", doGitClone(dstPath, u)) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", littleLFS)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Length) + little, big := standardCommitAndPushTest(t, dstPath) + littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath) + rawTest(t, &httpContext, little, big, littleLFS, bigLFS) + mediaTest(t, &httpContext, little, big, littleLFS, bigLFS) - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", big)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Length) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", bigLFS)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Length) - } - }) t.Run("BranchProtectMerge", doBranchProtectPRMerge(httpContext.Username, httpContext.Reponame, dstPath)) }) t.Run("SSH", func(t *testing.T) { @@ -177,115 +79,17 @@ func testGit(t *testing.T, u *url.URL) { dstPath, err := ioutil.TempDir("", sshContext.Reponame) assert.NoError(t, err) defer os.RemoveAll(dstPath) - var little, big, littleLFS, bigLFS string - - t.Run("Standard", func(t *testing.T) { - PrintCurrentTest(t) - t.Run("CreateRepo", doAPICreateRepository(sshContext, false)) - - //TODO get url from api - t.Run("Clone", doGitClone(dstPath, sshURL)) - - //time.Sleep(5 * time.Minute) - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - prefix := "data-file-" - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - little = commitAndPush(t, littleSize, dstPath, prefix) - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - return - } - PrintCurrentTest(t) - big = commitAndPush(t, bigSize, dstPath, prefix) - }) - }) - }) - t.Run("LFS", func(t *testing.T) { - PrintCurrentTest(t) - - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - //Setup git LFS - prefix := "lfs-data-file-" - _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("track", prefix+"*").RunInDir(dstPath) - assert.NoError(t, err) - err = git.AddChanges(dstPath, false, ".gitattributes") - assert.NoError(t, err) - - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - littleLFS = commitAndPush(t, littleSize, dstPath, prefix) - lockFileTest(t, littleLFS, dstPath) - - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - return - } - PrintCurrentTest(t) - bigLFS = commitAndPush(t, bigSize, dstPath, prefix) - lockFileTest(t, bigLFS, dstPath) - - }) - }) - t.Run("Locks", func(t *testing.T) { - PrintCurrentTest(t) - lockTest(t, u.String(), dstPath) - }) - }) - t.Run("Raw", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") - - // Request raw paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", little)) - resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", littleLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, littleSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", big)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, bigSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - } - }) - t.Run("Media", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") - - // Request media paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", little)) - resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", littleLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", big)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - } - }) + + t.Run("CreateRepo", doAPICreateRepository(sshContext, false)) + + //TODO get url from api + t.Run("Clone", doGitClone(dstPath, sshURL)) + + little, big := standardCommitAndPushTest(t, dstPath) + littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath) + rawTest(t, &sshContext, little, big, littleLFS, bigLFS) + mediaTest(t, &sshContext, little, big, littleLFS, bigLFS) + t.Run("BranchProtectMerge", doBranchProtectPRMerge(sshContext.Username, sshContext.Reponame, dstPath)) }) @@ -300,7 +104,115 @@ func ensureAnonymousClone(t *testing.T, u *url.URL) { } -func lockTest(t *testing.T, remote, repoPath string) { +func standardCommitAndPushTest(t *testing.T, dstPath string) (little, big string) { + t.Run("Standard", func(t *testing.T) { + PrintCurrentTest(t) + little, big = commitAndPushTest(t, dstPath, "data-file-") + }) + return +} + +func lfsCommitAndPushTest(t *testing.T, dstPath string) (littleLFS, bigLFS string) { + t.Run("LFS", func(t *testing.T) { + PrintCurrentTest(t) + prefix := "lfs-data-file-" + _, err := git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("track", prefix+"*").RunInDir(dstPath) + assert.NoError(t, err) + err = git.AddChanges(dstPath, false, ".gitattributes") + assert.NoError(t, err) + + littleLFS, bigLFS = commitAndPushTest(t, dstPath, prefix) + + t.Run("Locks", func(t *testing.T) { + PrintCurrentTest(t) + lockTest(t, dstPath) + }) + }) + return +} + +func commitAndPushTest(t *testing.T, dstPath, prefix string) (little, big string) { + t.Run("PushCommit", func(t *testing.T) { + PrintCurrentTest(t) + t.Run("Little", func(t *testing.T) { + PrintCurrentTest(t) + little = doCommitAndPush(t, littleSize, dstPath, prefix) + }) + t.Run("Big", func(t *testing.T) { + if testing.Short() { + return + } + PrintCurrentTest(t) + big = doCommitAndPush(t, bigSize, dstPath, prefix) + }) + }) + return +} + +func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS string) { + t.Run("Raw", func(t *testing.T) { + PrintCurrentTest(t) + username := ctx.Username + reponame := ctx.Reponame + + session := loginUser(t, username) + + // Request raw paths + req := NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", little)) + resp := session.MakeRequest(t, req, http.StatusOK) + assert.Equal(t, littleSize, resp.Body.Len()) + + req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", littleLFS)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.NotEqual(t, littleSize, resp.Body.Len()) + assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", big)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Body.Len()) + + req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", bigLFS)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.NotEqual(t, bigSize, resp.Body.Len()) + assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + } + }) +} + +func mediaTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS string) { + t.Run("Media", func(t *testing.T) { + PrintCurrentTest(t) + + username := ctx.Username + reponame := ctx.Reponame + + session := loginUser(t, username) + + // Request media paths + req := NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", little)) + resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, littleSize, resp.Length) + + req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", littleLFS)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, littleSize, resp.Length) + + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", big)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Length) + + req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", bigLFS)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Length) + } + }) +} + +func lockTest(t *testing.T, repoPath string) { lockFileTest(t, "README.md", repoPath) } @@ -315,7 +227,7 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } -func commitAndPush(t *testing.T, size int, repoPath, prefix string) string { +func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) _, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push From 8f4ab6faa130f603db74303024b4a016e499882e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 09:57:39 +0100 Subject: [PATCH 18/25] slight refactor --- integrations/git_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index c37ac4b12548..c211179a5968 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -60,7 +60,7 @@ func testGit(t *testing.T, u *url.URL) { rawTest(t, &httpContext, little, big, littleLFS, bigLFS) mediaTest(t, &httpContext, little, big, littleLFS, bigLFS) - t.Run("BranchProtectMerge", doBranchProtectPRMerge(httpContext.Username, httpContext.Reponame, dstPath)) + t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) }) t.Run("SSH", func(t *testing.T) { PrintCurrentTest(t) @@ -90,7 +90,7 @@ func testGit(t *testing.T, u *url.URL) { rawTest(t, &sshContext, little, big, littleLFS, bigLFS) mediaTest(t, &sshContext, little, big, littleLFS, bigLFS) - t.Run("BranchProtectMerge", doBranchProtectPRMerge(sshContext.Username, sshContext.Reponame, dstPath)) + t.Run("BranchProtectMerge", doBranchProtectPRMerge(&sshContext, dstPath)) }) }) @@ -273,13 +273,13 @@ func generateCommitWithNewData(size int, repoPath, email, fullName, prefix strin return filepath.Base(tmpFile.Name()), err } -func doBranchProtectPRMerge(username, reponame, dstPath string) func(t *testing.T) { +func doBranchProtectPRMerge(baseCtx *APITestContext, dstPath string) func(t *testing.T) { return func(t *testing.T) { PrintCurrentTest(t) t.Run("CreateBranchProtected", doGitCreateBranch(dstPath, "protected")) t.Run("PushProtectedBranch", doGitPushTestRepository(dstPath, "origin", "protected")) - ctx := NewAPITestContext(t, username, reponame) + ctx := NewAPITestContext(t, baseCtx.Username, baseCtx.Reponame) t.Run("ProtectProtectedBranchNoWhitelist", doProtectBranch(ctx, "protected", "")) t.Run("GenerateCommit", func(t *testing.T) { _, err := generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") @@ -290,12 +290,12 @@ func doBranchProtectPRMerge(username, reponame, dstPath string) func(t *testing. var pr api.PullRequest var err error t.Run("CreatePullRequest", func(t *testing.T) { - pr, err = doAPICreatePullRequest(ctx, username, reponame, "protected", "unprotected")(t) + pr, err = doAPICreatePullRequest(ctx, baseCtx.Username, baseCtx.Reponame, "protected", "unprotected")(t) assert.NoError(t, err) }) - t.Run("MergePR", doAPIMergePullRequest(ctx, username, reponame, pr.Index)) + t.Run("MergePR", doAPIMergePullRequest(ctx, baseCtx.Username, baseCtx.Reponame, pr.Index)) t.Run("PullProtected", doGitPull(dstPath, "origin", "protected")) - t.Run("ProtectProtectedBranchWhitelist", doProtectBranch(ctx, "protected", username)) + t.Run("ProtectProtectedBranchWhitelist", doProtectBranch(ctx, "protected", baseCtx.Username)) t.Run("CheckoutMaster", doGitCheckoutBranch(dstPath, "master")) t.Run("CreateBranchForced", doGitCreateBranch(dstPath, "toforce")) From 4863df6c8e683d212996950ea90c48015637f20d Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 10:10:32 +0100 Subject: [PATCH 19/25] Remove unnecessary "/" --- integrations/git_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index c211179a5968..e1795a6ae897 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -160,21 +160,21 @@ func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS s session := loginUser(t, username) // Request raw paths - req := NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", little)) + req := NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", little)) resp := session.MakeRequest(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", littleLFS)) + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", littleLFS)) resp = session.MakeRequest(t, req, http.StatusOK) assert.NotEqual(t, littleSize, resp.Body.Len()) assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", big)) + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", big)) resp = session.MakeRequest(t, req, http.StatusOK) assert.Equal(t, bigSize, resp.Body.Len()) - req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/raw/branch/master/", bigLFS)) + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", bigLFS)) resp = session.MakeRequest(t, req, http.StatusOK) assert.NotEqual(t, bigSize, resp.Body.Len()) assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) @@ -192,20 +192,20 @@ func mediaTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS session := loginUser(t, username) // Request media paths - req := NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", little)) + req := NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", little)) resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Length) - req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", littleLFS)) + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", littleLFS)) resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) assert.Equal(t, littleSize, resp.Length) if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", big)) + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", big)) resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) assert.Equal(t, bigSize, resp.Length) - req = NewRequest(t, "GET", path.Join("/", username, "/", reponame, "/media/branch/master/", bigLFS)) + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", bigLFS)) resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) assert.Equal(t, bigSize, resp.Length) } From 17ca9731500e12535f037947d5b205f8b5b21591 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 10:20:30 +0100 Subject: [PATCH 20/25] Restore ensureAnonymousClone --- integrations/git_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index e1795a6ae897..4874f6a26ff1 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -52,6 +52,7 @@ func testGit(t *testing.T, u *url.URL) { u.Path = httpContext.GitPath() u.User = url.UserPassword(username, userPassword) + ensureAnonymousClone(t, u) t.Run("Clone", doGitClone(dstPath, u)) @@ -70,7 +71,6 @@ func testGit(t *testing.T, u *url.URL) { //Setup key the user ssh key withKeyFile(t, keyname, func(keyFile string) { t.Run("CreateUserKey", doAPICreateUserKey(sshContext, "test-key", keyFile)) - PrintCurrentTest(t) //Setup remote link sshURL := createSSHUrl(sshContext.GitPath(), u) From 2ca27f341d1332348ca04211a4516accc603e760 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 10:24:13 +0100 Subject: [PATCH 21/25] Restore ensureAnonymousClone --- integrations/git_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index 4874f6a26ff1..1fcb16f6c25b 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -49,10 +49,10 @@ func testGit(t *testing.T, u *url.URL) { defer os.RemoveAll(dstPath) t.Run("CreateRepo", doAPICreateRepository(httpContext, false)) + ensureAnonymousClone(t, u) u.Path = httpContext.GitPath() u.User = url.UserPassword(username, userPassword) - ensureAnonymousClone(t, u) t.Run("Clone", doGitClone(dstPath, u)) @@ -73,6 +73,7 @@ func testGit(t *testing.T, u *url.URL) { t.Run("CreateUserKey", doAPICreateUserKey(sshContext, "test-key", keyFile)) //Setup remote link + //TODO: get url from api sshURL := createSSHUrl(sshContext.GitPath(), u) //Setup clone folder @@ -82,7 +83,6 @@ func testGit(t *testing.T, u *url.URL) { t.Run("CreateRepo", doAPICreateRepository(sshContext, false)) - //TODO get url from api t.Run("Clone", doGitClone(dstPath, sshURL)) little, big := standardCommitAndPushTest(t, dstPath) From 86f98cdb3b4ca927845eb1d594e8a6aee791a25a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 12:16:28 +0100 Subject: [PATCH 22/25] Run rev-list on server side --- cmd/hook.go | 16 +++++----------- modules/private/hook.go | 25 +++++++++++++++---------- routers/private/hook.go | 27 +++++++++++++++++++++------ 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index f8aa32f33c56..b4225dbcbf2a 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -86,20 +86,14 @@ func runHookPreReceive(c *cli.Context) error { newCommitID := string(fields[1]) refFullName := string(fields[2]) - output, errErr := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).Run() - err := "" - if errErr != nil { - err = errErr.Error() - } // If the ref is a branch, check if it's protected if strings.HasPrefix(refFullName, git.BranchPrefix) { statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{ - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - RefFullName: refFullName, - UserID: userID, - Output: output, - Err: err, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + RefFullName: refFullName, + UserID: userID, + GitObjectDirectory: os.Getenv(private.GitObjectDirectory), }) switch statusCode { case http.StatusInternalServerError: diff --git a/modules/private/hook.go b/modules/private/hook.go index 6bf7145f9bdc..4c557182c50d 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -13,28 +13,33 @@ import ( "code.gitea.io/gitea/modules/setting" ) +// Git environment variables +const ( + GitAlternativeObjectDirectories = "GIT_ALTERNATE_OBJECT_DIRECTORIES" + GitObjectDirectory = "GIT_OBJECT_DIRECTORY" + GitQuarantinePath = "GIT_QUARANTINE_PATH" +) + // HookOptions represents the options for the Hook calls type HookOptions struct { - OldCommitID string - NewCommitID string - RefFullName string - UserID int64 - UserName string - Output string - Err string + OldCommitID string + NewCommitID string + RefFullName string + UserID int64 + UserName string + GitObjectDirectory string } // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&output=%s&err=%s", + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s", url.PathEscape(ownerName), url.PathEscape(repoName), url.QueryEscape(opts.OldCommitID), url.QueryEscape(opts.NewCommitID), url.QueryEscape(opts.RefFullName), opts.UserID, - url.QueryEscape(opts.Output), - url.QueryEscape(opts.Err)) + url.QueryEscape(opts.GitObjectDirectory)) resp, err := newInternalRequest(reqURL, "GET").Response() if err != nil { diff --git a/routers/private/hook.go b/routers/private/hook.go index 46152a130141..073f3d7d1599 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -8,11 +8,14 @@ package private import ( "fmt" "net/http" + "os" + "path/filepath" "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/util" macaron "gopkg.in/macaron.v1" @@ -26,8 +29,7 @@ func HookPreReceive(ctx *macaron.Context) { newCommitID := ctx.QueryTrim("new") refFullName := ctx.QueryTrim("ref") userID := ctx.QueryInt64("userID") - revListOutput := ctx.Query("output") - revListErr := ctx.Query("err") + gitObjectDirectory := ctx.QueryTrim("gitObjectDirectory") branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) @@ -59,13 +61,26 @@ func HookPreReceive(ctx *macaron.Context) { // detect force push if git.EmptySHA != oldCommitID { - if revListErr != "" { - log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, revListErr) + alternativeObjectDirectories := os.Getenv(private.GitAlternativeObjectDirectories) + if len(alternativeObjectDirectories) > 0 { + alternativeObjectDirectories += ":" + } + alternativeObjectDirectories += "\"" + filepath.Join(repo.RepoPath(), "objects") + "\"" + + env := append(os.Environ(), + private.GitAlternativeObjectDirectories+"="+alternativeObjectDirectories, + private.GitObjectDirectory+"="+gitObjectDirectory, + private.GitQuarantinePath+"="+gitObjectDirectory, + ) + + output, err := git.NewCommand("rev-list", "--max-count=1", oldCommitID, "^"+newCommitID).RunInDirWithEnv(repo.RepoPath(), env) + if err != nil { + log.Error("Unable to detect force push between: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": fmt.Sprintf("Fail to detect force push: %v", revListErr), + "err": fmt.Sprintf("Fail to detect force push: %v", err), }) return - } else if len(revListOutput) > 0 { + } else if len(output) > 0 { log.Warn("Forbidden: Branch: %s in %-v is protected from force push", branchName, repo) ctx.JSON(http.StatusForbidden, map[string]interface{}{ "err": fmt.Sprintf("branch %s is protected from force push", branchName), From b1aebfea5f2a721ee24b187c803218eafef0c330 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 12:45:58 +0100 Subject: [PATCH 23/25] Try passing in the alternative directories instead --- cmd/hook.go | 11 ++++++----- modules/private/hook.go | 19 +++++++++++-------- routers/private/hook.go | 10 ++-------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/cmd/hook.go b/cmd/hook.go index b4225dbcbf2a..b3e900afee4d 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -89,11 +89,12 @@ func runHookPreReceive(c *cli.Context) error { // If the ref is a branch, check if it's protected if strings.HasPrefix(refFullName, git.BranchPrefix) { statusCode, msg := private.HookPreReceive(username, reponame, private.HookOptions{ - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - RefFullName: refFullName, - UserID: userID, - GitObjectDirectory: os.Getenv(private.GitObjectDirectory), + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + RefFullName: refFullName, + UserID: userID, + GitAlternativeObjectDirectories: os.Getenv(private.GitAlternativeObjectDirectories), + GitObjectDirectory: os.Getenv(private.GitObjectDirectory), }) switch statusCode { case http.StatusInternalServerError: diff --git a/modules/private/hook.go b/modules/private/hook.go index 4c557182c50d..7e2a475d4b8a 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -22,24 +22,27 @@ const ( // HookOptions represents the options for the Hook calls type HookOptions struct { - OldCommitID string - NewCommitID string - RefFullName string - UserID int64 - UserName string - GitObjectDirectory string + OldCommitID string + NewCommitID string + RefFullName string + UserID int64 + UserName string + GitObjectDirectory string + GitAlternativeObjectDirectories string } // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s", + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s&gitAlternativeObjectDirectories=%s", url.PathEscape(ownerName), url.PathEscape(repoName), url.QueryEscape(opts.OldCommitID), url.QueryEscape(opts.NewCommitID), url.QueryEscape(opts.RefFullName), opts.UserID, - url.QueryEscape(opts.GitObjectDirectory)) + url.QueryEscape(opts.GitObjectDirectory), + url.QueryEscape(opts.GitAlternativeObjectDirectories), + ) resp, err := newInternalRequest(reqURL, "GET").Response() if err != nil { diff --git a/routers/private/hook.go b/routers/private/hook.go index 073f3d7d1599..700c8bf33279 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -9,7 +9,6 @@ import ( "fmt" "net/http" "os" - "path/filepath" "strings" "code.gitea.io/gitea/models" @@ -30,6 +29,7 @@ func HookPreReceive(ctx *macaron.Context) { refFullName := ctx.QueryTrim("ref") userID := ctx.QueryInt64("userID") gitObjectDirectory := ctx.QueryTrim("gitObjectDirectory") + gitAlternativeObjectDirectories := ctx.QueryTrim("gitAlternativeObjectDirectories") branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) @@ -61,14 +61,8 @@ func HookPreReceive(ctx *macaron.Context) { // detect force push if git.EmptySHA != oldCommitID { - alternativeObjectDirectories := os.Getenv(private.GitAlternativeObjectDirectories) - if len(alternativeObjectDirectories) > 0 { - alternativeObjectDirectories += ":" - } - alternativeObjectDirectories += "\"" + filepath.Join(repo.RepoPath(), "objects") + "\"" - env := append(os.Environ(), - private.GitAlternativeObjectDirectories+"="+alternativeObjectDirectories, + private.GitAlternativeObjectDirectories+"="+gitAlternativeObjectDirectories, private.GitObjectDirectory+"="+gitObjectDirectory, private.GitQuarantinePath+"="+gitObjectDirectory, ) From 5209e94587c6b61a5e0d31466ff9e0ac04148876 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 May 2019 16:05:20 +0100 Subject: [PATCH 24/25] Mark test as skipped --- integrations/git_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/git_test.go b/integrations/git_test.go index 1fcb16f6c25b..ce5aee493d83 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -142,6 +142,7 @@ func commitAndPushTest(t *testing.T, dstPath, prefix string) (little, big string }) t.Run("Big", func(t *testing.T) { if testing.Short() { + t.Skip("Skipping test in short mode.") return } PrintCurrentTest(t) From 8dbb91a79187d2616e37765cd812c5e7f80d2881 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 20 May 2019 14:28:00 +0100 Subject: [PATCH 25/25] Improve git test * Ensure that the lfs files are created with a different prefix * Reduce the replication in git_test.go * Remove unnecessary "/" --- .../api_helper_for_declarative_test.go | 42 ++ .../git_helper_for_declarative_test.go | 36 +- integrations/git_test.go | 426 +++++++++--------- 3 files changed, 280 insertions(+), 224 deletions(-) diff --git a/integrations/api_helper_for_declarative_test.go b/integrations/api_helper_for_declarative_test.go index 943981ead2f0..85f0ab621f84 100644 --- a/integrations/api_helper_for_declarative_test.go +++ b/integrations/api_helper_for_declarative_test.go @@ -5,11 +5,14 @@ package integrations import ( + "encoding/json" "fmt" "io/ioutil" "net/http" "testing" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" api "code.gitea.io/gitea/modules/structs" "github.com/stretchr/testify/assert" ) @@ -150,3 +153,42 @@ func doAPICreateDeployKey(ctx APITestContext, keyname, keyFile string, readOnly ctx.Session.MakeRequest(t, req, http.StatusCreated) } } + +func doAPICreatePullRequest(ctx APITestContext, owner, repo, baseBranch, headBranch string) func(*testing.T) (api.PullRequest, error) { + return func(t *testing.T) (api.PullRequest, error) { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/pulls?token=%s", + owner, repo, ctx.Token) + req := NewRequestWithJSON(t, http.MethodPost, urlStr, &api.CreatePullRequestOption{ + Head: headBranch, + Base: baseBranch, + Title: fmt.Sprintf("create a pr from %s to %s", headBranch, baseBranch), + }) + + expected := 201 + if ctx.ExpectedCode != 0 { + expected = ctx.ExpectedCode + } + resp := ctx.Session.MakeRequest(t, req, expected) + decoder := json.NewDecoder(resp.Body) + pr := api.PullRequest{} + err := decoder.Decode(&pr) + return pr, err + } +} + +func doAPIMergePullRequest(ctx APITestContext, owner, repo string, index int64) func(*testing.T) { + return func(t *testing.T) { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/merge?token=%s", + owner, repo, index, ctx.Token) + req := NewRequestWithJSON(t, http.MethodPost, urlStr, &auth.MergePullRequestForm{ + MergeMessageField: "doAPIMergePullRequest Merge", + Do: string(models.MergeStyleMerge), + }) + + if ctx.ExpectedCode != 0 { + ctx.Session.MakeRequest(t, req, ctx.ExpectedCode) + return + } + ctx.Session.MakeRequest(t, req, 200) + } +} diff --git a/integrations/git_helper_for_declarative_test.go b/integrations/git_helper_for_declarative_test.go index b4fead66253b..235f4b4a9b74 100644 --- a/integrations/git_helper_for_declarative_test.go +++ b/integrations/git_helper_for_declarative_test.go @@ -112,16 +112,44 @@ func doGitAddRemote(dstPath, remoteName string, u *url.URL) func(*testing.T) { } } -func doGitPushTestRepository(dstPath, remoteName, branch string) func(*testing.T) { +func doGitPushTestRepository(dstPath string, args ...string) func(*testing.T) { return func(t *testing.T) { - _, err := git.NewCommand("push", "-u", remoteName, branch).RunInDir(dstPath) + _, err := git.NewCommand(append([]string{"push", "-u"}, args...)...).RunInDir(dstPath) assert.NoError(t, err) } } -func doGitPushTestRepositoryFail(dstPath, remoteName, branch string) func(*testing.T) { +func doGitPushTestRepositoryFail(dstPath string, args ...string) func(*testing.T) { return func(t *testing.T) { - _, err := git.NewCommand("push", "-u", remoteName, branch).RunInDir(dstPath) + _, err := git.NewCommand(append([]string{"push"}, args...)...).RunInDir(dstPath) assert.Error(t, err) } } + +func doGitCreateBranch(dstPath, branch string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand("checkout", "-b", branch).RunInDir(dstPath) + assert.NoError(t, err) + } +} + +func doGitCheckoutBranch(dstPath string, args ...string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand(append([]string{"checkout"}, args...)...).RunInDir(dstPath) + assert.NoError(t, err) + } +} + +func doGitMerge(dstPath string, args ...string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand(append([]string{"merge"}, args...)...).RunInDir(dstPath) + assert.NoError(t, err) + } +} + +func doGitPull(dstPath string, args ...string) func(*testing.T) { + return func(t *testing.T) { + _, err := git.NewCommand(append([]string{"pull"}, args...)...).RunInDir(dstPath) + assert.NoError(t, err) + } +} diff --git a/integrations/git_test.go b/integrations/git_test.go index 0554f9a5aead..ce5aee493d83 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -13,11 +13,13 @@ import ( "os" "path" "path/filepath" + "strconv" "testing" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + api "code.gitea.io/gitea/modules/structs" "github.com/stretchr/testify/assert" ) @@ -43,119 +45,23 @@ func testGit(t *testing.T, u *url.URL) { httpContext.Reponame = "repo-tmp-17" dstPath, err := ioutil.TempDir("", httpContext.Reponame) - var little, big, littleLFS, bigLFS string - assert.NoError(t, err) defer os.RemoveAll(dstPath) - t.Run("Standard", func(t *testing.T) { - PrintCurrentTest(t) - ensureAnonymousClone(t, u) - - t.Run("CreateRepo", doAPICreateRepository(httpContext, false)) - - u.Path = httpContext.GitPath() - u.User = url.UserPassword(username, userPassword) - - t.Run("Clone", doGitClone(dstPath, u)) - - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - little = commitAndPush(t, littleSize, dstPath) - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - return - } - PrintCurrentTest(t) - big = commitAndPush(t, bigSize, dstPath) - }) - }) - }) - t.Run("LFS", func(t *testing.T) { - PrintCurrentTest(t) - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - //Setup git LFS - _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath) - assert.NoError(t, err) - err = git.AddChanges(dstPath, false, ".gitattributes") - assert.NoError(t, err) - - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - littleLFS = commitAndPush(t, littleSize, dstPath) - lockFileTest(t, littleLFS, dstPath) - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - return - } - PrintCurrentTest(t) - bigLFS = commitAndPush(t, bigSize, dstPath) - lockFileTest(t, bigLFS, dstPath) - }) - }) - t.Run("Locks", func(t *testing.T) { - PrintCurrentTest(t) - lockTest(t, u.String(), dstPath) - }) - }) - t.Run("Raw", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") - - // Request raw paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", little)) - resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", littleLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, littleSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", big)) - nilResp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, nilResp.Length) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, bigSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - } - - }) - t.Run("Media", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") - // Request media paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", little)) - resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Length) + t.Run("CreateRepo", doAPICreateRepository(httpContext, false)) + ensureAnonymousClone(t, u) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", littleLFS)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Length) + u.Path = httpContext.GitPath() + u.User = url.UserPassword(username, userPassword) - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", big)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Length) + t.Run("Clone", doGitClone(dstPath, u)) - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", bigLFS)) - resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Length) - } - }) + little, big := standardCommitAndPushTest(t, dstPath) + littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath) + rawTest(t, &httpContext, little, big, littleLFS, bigLFS) + mediaTest(t, &httpContext, little, big, littleLFS, bigLFS) + t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) }) t.Run("SSH", func(t *testing.T) { PrintCurrentTest(t) @@ -165,123 +71,26 @@ func testGit(t *testing.T, u *url.URL) { //Setup key the user ssh key withKeyFile(t, keyname, func(keyFile string) { t.Run("CreateUserKey", doAPICreateUserKey(sshContext, "test-key", keyFile)) - PrintCurrentTest(t) //Setup remote link + //TODO: get url from api sshURL := createSSHUrl(sshContext.GitPath(), u) //Setup clone folder dstPath, err := ioutil.TempDir("", sshContext.Reponame) assert.NoError(t, err) defer os.RemoveAll(dstPath) - var little, big, littleLFS, bigLFS string - - t.Run("Standard", func(t *testing.T) { - PrintCurrentTest(t) - t.Run("CreateRepo", doAPICreateRepository(sshContext, false)) - - //TODO get url from api - t.Run("Clone", doGitClone(dstPath, sshURL)) - - //time.Sleep(5 * time.Minute) - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - little = commitAndPush(t, littleSize, dstPath) - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - return - } - PrintCurrentTest(t) - big = commitAndPush(t, bigSize, dstPath) - }) - }) - }) - t.Run("LFS", func(t *testing.T) { - PrintCurrentTest(t) - t.Run("PushCommit", func(t *testing.T) { - PrintCurrentTest(t) - //Setup git LFS - _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath) - assert.NoError(t, err) - err = git.AddChanges(dstPath, false, ".gitattributes") - assert.NoError(t, err) - - t.Run("Little", func(t *testing.T) { - PrintCurrentTest(t) - littleLFS = commitAndPush(t, littleSize, dstPath) - lockFileTest(t, littleLFS, dstPath) - - }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - return - } - PrintCurrentTest(t) - bigLFS = commitAndPush(t, bigSize, dstPath) - lockFileTest(t, bigLFS, dstPath) - - }) - }) - t.Run("Locks", func(t *testing.T) { - PrintCurrentTest(t) - lockTest(t, u.String(), dstPath) - }) - }) - t.Run("Raw", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") - - // Request raw paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", little)) - resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", littleLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, littleSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", big)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.NotEqual(t, bigSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) - } - }) - t.Run("Media", func(t *testing.T) { - PrintCurrentTest(t) - session := loginUser(t, "user2") - - // Request media paths - req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", little)) - resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", littleLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, littleSize, resp.Body.Len()) - - if !testing.Short() { - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", big)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - - req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", bigLFS)) - resp = session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, bigSize, resp.Body.Len()) - } - }) + t.Run("CreateRepo", doAPICreateRepository(sshContext, false)) + + t.Run("Clone", doGitClone(dstPath, sshURL)) + + little, big := standardCommitAndPushTest(t, dstPath) + littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath) + rawTest(t, &sshContext, little, big, littleLFS, bigLFS) + mediaTest(t, &sshContext, little, big, littleLFS, bigLFS) + + t.Run("BranchProtectMerge", doBranchProtectPRMerge(&sshContext, dstPath)) }) }) @@ -295,7 +104,116 @@ func ensureAnonymousClone(t *testing.T, u *url.URL) { } -func lockTest(t *testing.T, remote, repoPath string) { +func standardCommitAndPushTest(t *testing.T, dstPath string) (little, big string) { + t.Run("Standard", func(t *testing.T) { + PrintCurrentTest(t) + little, big = commitAndPushTest(t, dstPath, "data-file-") + }) + return +} + +func lfsCommitAndPushTest(t *testing.T, dstPath string) (littleLFS, bigLFS string) { + t.Run("LFS", func(t *testing.T) { + PrintCurrentTest(t) + prefix := "lfs-data-file-" + _, err := git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("track", prefix+"*").RunInDir(dstPath) + assert.NoError(t, err) + err = git.AddChanges(dstPath, false, ".gitattributes") + assert.NoError(t, err) + + littleLFS, bigLFS = commitAndPushTest(t, dstPath, prefix) + + t.Run("Locks", func(t *testing.T) { + PrintCurrentTest(t) + lockTest(t, dstPath) + }) + }) + return +} + +func commitAndPushTest(t *testing.T, dstPath, prefix string) (little, big string) { + t.Run("PushCommit", func(t *testing.T) { + PrintCurrentTest(t) + t.Run("Little", func(t *testing.T) { + PrintCurrentTest(t) + little = doCommitAndPush(t, littleSize, dstPath, prefix) + }) + t.Run("Big", func(t *testing.T) { + if testing.Short() { + t.Skip("Skipping test in short mode.") + return + } + PrintCurrentTest(t) + big = doCommitAndPush(t, bigSize, dstPath, prefix) + }) + }) + return +} + +func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS string) { + t.Run("Raw", func(t *testing.T) { + PrintCurrentTest(t) + username := ctx.Username + reponame := ctx.Reponame + + session := loginUser(t, username) + + // Request raw paths + req := NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", little)) + resp := session.MakeRequest(t, req, http.StatusOK) + assert.Equal(t, littleSize, resp.Body.Len()) + + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", littleLFS)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.NotEqual(t, littleSize, resp.Body.Len()) + assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", big)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Body.Len()) + + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", bigLFS)) + resp = session.MakeRequest(t, req, http.StatusOK) + assert.NotEqual(t, bigSize, resp.Body.Len()) + assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + } + }) +} + +func mediaTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS string) { + t.Run("Media", func(t *testing.T) { + PrintCurrentTest(t) + + username := ctx.Username + reponame := ctx.Reponame + + session := loginUser(t, username) + + // Request media paths + req := NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", little)) + resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, littleSize, resp.Length) + + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", littleLFS)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, littleSize, resp.Length) + + if !testing.Short() { + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", big)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Length) + + req = NewRequest(t, "GET", path.Join("/", username, reponame, "/media/branch/master/", bigLFS)) + resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK) + assert.Equal(t, bigSize, resp.Length) + } + }) +} + +func lockTest(t *testing.T, repoPath string) { lockFileTest(t, "README.md", repoPath) } @@ -310,22 +228,22 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } -func commitAndPush(t *testing.T, size int, repoPath string) string { - name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two") +func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { + name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) - _, err = git.NewCommand("push").RunInDir(repoPath) //Push + _, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push assert.NoError(t, err) return name } -func generateCommitWithNewData(size int, repoPath, email, fullName string) (string, error) { +func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) { //Generate random file data := make([]byte, size) _, err := rand.Read(data) if err != nil { return "", err } - tmpFile, err := ioutil.TempFile(repoPath, "data-file-") + tmpFile, err := ioutil.TempFile(repoPath, prefix) if err != nil { return "", err } @@ -355,3 +273,71 @@ func generateCommitWithNewData(size int, repoPath, email, fullName string) (stri }) return filepath.Base(tmpFile.Name()), err } + +func doBranchProtectPRMerge(baseCtx *APITestContext, dstPath string) func(t *testing.T) { + return func(t *testing.T) { + PrintCurrentTest(t) + t.Run("CreateBranchProtected", doGitCreateBranch(dstPath, "protected")) + t.Run("PushProtectedBranch", doGitPushTestRepository(dstPath, "origin", "protected")) + + ctx := NewAPITestContext(t, baseCtx.Username, baseCtx.Reponame) + t.Run("ProtectProtectedBranchNoWhitelist", doProtectBranch(ctx, "protected", "")) + t.Run("GenerateCommit", func(t *testing.T) { + _, err := generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") + assert.NoError(t, err) + }) + t.Run("FailToPushToProtectedBranch", doGitPushTestRepositoryFail(dstPath, "origin", "protected")) + t.Run("PushToUnprotectedBranch", doGitPushTestRepository(dstPath, "origin", "protected:unprotected")) + var pr api.PullRequest + var err error + t.Run("CreatePullRequest", func(t *testing.T) { + pr, err = doAPICreatePullRequest(ctx, baseCtx.Username, baseCtx.Reponame, "protected", "unprotected")(t) + assert.NoError(t, err) + }) + t.Run("MergePR", doAPIMergePullRequest(ctx, baseCtx.Username, baseCtx.Reponame, pr.Index)) + t.Run("PullProtected", doGitPull(dstPath, "origin", "protected")) + t.Run("ProtectProtectedBranchWhitelist", doProtectBranch(ctx, "protected", baseCtx.Username)) + + t.Run("CheckoutMaster", doGitCheckoutBranch(dstPath, "master")) + t.Run("CreateBranchForced", doGitCreateBranch(dstPath, "toforce")) + t.Run("GenerateCommit", func(t *testing.T) { + _, err := generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") + assert.NoError(t, err) + }) + t.Run("FailToForcePushToProtectedBranch", doGitPushTestRepositoryFail(dstPath, "-f", "origin", "toforce:protected")) + t.Run("MergeProtectedToToforce", doGitMerge(dstPath, "protected")) + t.Run("PushToProtectedBranch", doGitPushTestRepository(dstPath, "origin", "toforce:protected")) + t.Run("CheckoutMasterAgain", doGitCheckoutBranch(dstPath, "master")) + } +} + +func doProtectBranch(ctx APITestContext, branch string, userToWhitelist string) func(t *testing.T) { + // We are going to just use the owner to set the protection. + return func(t *testing.T) { + csrf := GetCSRF(t, ctx.Session, fmt.Sprintf("/%s/%s/settings/branches", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame))) + + if userToWhitelist == "" { + // Change branch to protected + req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings/branches/%s", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame), url.PathEscape(branch)), map[string]string{ + "_csrf": csrf, + "protected": "on", + }) + ctx.Session.MakeRequest(t, req, http.StatusFound) + } else { + user, err := models.GetUserByName(userToWhitelist) + assert.NoError(t, err) + // Change branch to protected + req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings/branches/%s", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame), url.PathEscape(branch)), map[string]string{ + "_csrf": csrf, + "protected": "on", + "enable_whitelist": "on", + "whitelist_users": strconv.FormatInt(user.ID, 10), + }) + ctx.Session.MakeRequest(t, req, http.StatusFound) + } + // Check if master branch has been locked successfully + flashCookie := ctx.Session.GetCookie("macaron_flash") + assert.NotNil(t, flashCookie) + assert.EqualValues(t, "success%3DBranch%2Bprotection%2Bfor%2Bbranch%2B%2527"+url.QueryEscape(branch)+"%2527%2Bhas%2Bbeen%2Bupdated.", flashCookie.Value) + } +}