Limit repo size

custom/conf/app.example.ini

@@ -957,6 +957,15 @@ LEVEL = Info
 
 ;; Allow fork repositories without maximum number limit
 ;ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT = true
+;
+;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit
+;; "false" means no limit will be enforced, even if specified on a repository
+;ENABLE_SIZE_LIMIT = false
+;
+;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit
+;; If repository has it's own limit set in UI it will override the global setting
+;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ...
+;REPO_SIZE_LIMIT = 0
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/doc/administration/config-cheat-sheet.en-us.md

@@ -114,6 +114,9 @@ In addition there is _`StaticRootPath`_ which can be set as a built-in at build
 - `ALLOW_DELETION_OF_UNADOPTED_REPOSITORIES`: **false**: Allow non-admin users to delete unadopted repositories
 - `DISABLE_DOWNLOAD_SOURCE_ARCHIVES`: **false**: Don't allow download source archive files from UI
 - `ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT`: **true**: Allow fork repositories without maximum number limit
+- `ENABLE_SIZE_LIMIT`: **false**: Enable global per repository size
+limit defined in `REPO_SIZE_LIMIT` or at the repository itself.
+- `REPO_SIZE_LIMIT`: **0**: The global default size limit to apply per repository in bytes when `ENABLE_SIZE_LIMIT` is `true`. Value `0` - no limit. The repository limit set via user interface by admin takes precedence over this value. Standard units of measurements can be used with this number like - `B`, `KB`, `KiB`, `MB`, `MiB`, .... , `EB`, `EiB`, ...
 
 ### Repository - Editor (`repository.editor`)
 

docs/content/doc/administration/repo-size-limit.en-us.md

@@ -0,0 +1,52 @@
+---
+date: "2023-05-24T13:00:00+00:00"
+title: "Per repository size limit"
+slug: "repo-size-limit"
+weight: 12
+toc: false
+draft: false
+aliases:
+  - /en-us/repo-size-limit
+menu:
+  sidebar:
+    parent: "administration"
+    name: "Per repository size limit"
+    weight: 12
+    identifier: "repo-size-limit"
+---
+
+# Gitea per repository size limit setup
+
+To use Gitea's experimental built-in per repository size limit support, Administrator must update the `app.ini` file:
+
+```ini
+;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit
+;; "false" means no limit will be enforced, even if specified on a repository
+ENABLE_SIZE_LIMIT = true
+
+;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit
+;; If repository has it's own limit set in UI it will override the global setting
+;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ...
+REPO_SIZE_LIMIT = 500 MB
+
+This setting is persistent.
+
+The size limitation is triggered when repository `disk size` + `new commit size` > `defined repository size limit`
+
+If size limitation is triggered the feature would prevent commits that increase repository size on disk
+of gitea server and allow those that decrease it
+
+# Gitea per repository size limit setup in UI
+
+1. For Gitea admin it is possible during runtime to enable/disable limit size feature, change the global size limit on the fly.
+**This setting is not persistent across restarts**
+
+`Admin panel/Site settings` -> `Repository management`
+
+Persistance can be achieved if the limit is maintained by editing `app.ini` file
+
+2. The individually set per repository limit in `Settings` of the
+repository would take precedence over global limit when the size limit
+feature is enabled. Only admin can modify those limits
+
+**Note**: Size checking for large repositories is time consuming operation so time of push under size limit might increase up to a minute depending on your server hardware

docs/content/page/index.en-us.md

@@ -112,6 +112,7 @@ You can try it out using [the online demo](https://try.gitea.io/).
       - Hooks
     - Repository management
       - See all repository information and manage repositories
+      - Set per repository global size limit, disable/enable the repository limit (runtime only), permanent should be done via Configuration file
     - Authentication sources
       - OAuth
       - PAM
@@ -170,6 +171,7 @@ You can try it out using [the online demo](https://try.gitea.io/).
   - Git LFS
   - Watch, Star, Fork
   - View watchers, stars, and forks
+  - Size limit (excl. LFS)
   - Code
     - Branch browser
     - Web based file upload and creation

models/migrations/migrations.go

@@ -497,6 +497,8 @@ var migrations = []Migration{
 	NewMigration("Add PinOrder Column", v1_20.AddPinOrderToIssue),
 	// v259 -> 260
 	NewMigration("Convert scoped access tokens", v1_20.ConvertScopedAccessTokens),
+	// to modify later
+	NewMigration("Add size limit on repository", v1_20.AddSizeLimitOnRepo),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_20/v999.go

@@ -0,0 +1,17 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_20 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddSizeLimitOnRepo(x *xorm.Engine) error {
+	type Repository struct {
+		ID        int64 `xorm:"pk autoincr"`
+		SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
+	}
+
+	return x.Sync2(new(Repository))
+}

models/repo/repo.go

@@ -162,7 +162,9 @@ type Repository struct {
 	BaseRepo                        *Repository        `xorm:"-"`
 	IsTemplate                      bool               `xorm:"INDEX NOT NULL DEFAULT false"`
 	TemplateID                      int64              `xorm:"INDEX"`
+	SizeLimit                       int64              `xorm:"NOT NULL DEFAULT 0"`
 	Size                            int64              `xorm:"NOT NULL DEFAULT 0"`
+	EnableSizeLimit                 bool               `xorm:"NOT NULL DEFAULT true"`
 	CodeIndexerStatus               *RepoIndexerStatus `xorm:"-"`
 	StatsIndexerStatus              *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
@@ -494,6 +496,27 @@ func (repo *Repository) IsOwnedBy(userID int64) bool {
 	return repo.OwnerID == userID
 }
 
+// GetActualSizeLimit returns repository size limit in bytes
+// or global repository limit setting if per repository size limit is not set
+func (repo *Repository) GetActualSizeLimit() int64 {
+	sizeLimit := repo.SizeLimit
+	if setting.RepoSizeLimit > 0 && sizeLimit == 0 {
+		sizeLimit = setting.RepoSizeLimit
+	}
+	return sizeLimit
+}
+
+// RepoSizeIsOversized return true if is over size limitation
+func (repo *Repository) IsRepoSizeOversized(additionalSize int64) bool {
+	return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.Size+additionalSize > repo.GetActualSizeLimit()
+}
+
+// RepoSizeLimitEnabled return true if size limit checking is enabled and limit is non zero for this specific repository
+// this is used to enable size checking during pre-receive hook
+func (repo *Repository) IsRepoSizeLimitEnabled() bool {
+	return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0
+}
+
 // CanCreateBranch returns true if repository meets the requirements for creating new branches.
 func (repo *Repository) CanCreateBranch() bool {
 	return !repo.IsMirror

modules/base/tool.go

@@ -141,6 +141,13 @@ func FileSize(s int64) string {
 	return humanize.IBytes(uint64(s))
 }
 
+// Get FileSize bytes value from  String.
+func GetFileSize(s string) (int64, error) {
+	v, err := humanize.ParseBytes(s)
+	iv := int64(v)
+	return iv, err
+}
+
 // EllipsisString returns a truncated short string,
 // it appends '...' in the end of the length of string is too large.
 func EllipsisString(str string, length int) string {

modules/base/tool_test.go

@@ -114,6 +114,13 @@ func TestFileSize(t *testing.T) {
 	assert.Equal(t, "2.0 EiB", FileSize(size))
 }
 
+func TestGetFileSize(t *testing.T) {
+	var size int64 = 512 * 1024 * 1024 * 1024
+	s, err := GetFileSize("512 GiB")
+	assert.Equal(t, s, size)
+	assert.Nil(t, err)
+}
+
 func TestEllipsisString(t *testing.T) {
 	assert.Equal(t, "...", EllipsisString("foobar", 0))
 	assert.Equal(t, "...", EllipsisString("foobar", 1))

modules/git/repo.go

@@ -227,6 +227,74 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error {
 	return nil
 }
 
+// CountObject represents repository count objects report
+type CountObject struct {
+	Count       int64
+	Size        int64
+	InPack      int64
+	Packs       int64
+	SizePack    int64
+	PrunePack   int64
+	Garbage     int64
+	SizeGarbage int64
+}
+
+const (
+	statCount        = "count: "
+	statSize         = "size: "
+	statInpack       = "in-pack: "
+	statPacks        = "packs: "
+	statSizePack     = "size-pack: "
+	statPrunePackage = "prune-package: "
+	statGarbage      = "garbage: "
+	statSizeGarbage  = "size-garbage: "
+)
+
+// CountObjects returns the results of git count-objects on the repoPath
+func CountObjects(ctx context.Context, repoPath string) (*CountObject, error) {
+	return CountObjectsWithEnv(ctx, repoPath, nil)
+}
+
+// CountObjectsWithEnv returns the results of git count-objects on the repoPath with custom env setup
+func CountObjectsWithEnv(ctx context.Context, repoPath string, env []string) (*CountObject, error) {
+	cmd := NewCommand(ctx, "count-objects", "-v")
+	stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath, Env: env})
+	if err != nil {
+		return nil, err
+	}
+
+	return parseSize(stdout), nil
+}
+
+// parseSize parses the output from count-objects and return a CountObject
+func parseSize(objects string) *CountObject {
+	repoSize := new(CountObject)
+	for _, line := range strings.Split(objects, "\n") {
+		switch {
+		case strings.HasPrefix(line, statCount):
+			repoSize.Count, _ = strconv.ParseInt(line[7:], 10, 64)
+		case strings.HasPrefix(line, statSize):
+			number, _ := strconv.ParseInt(line[6:], 10, 64)
+			repoSize.Size = number * 1024
+		case strings.HasPrefix(line, statInpack):
+			repoSize.InPack, _ = strconv.ParseInt(line[9:], 10, 64)
+		case strings.HasPrefix(line, statPacks):
+			repoSize.Packs, _ = strconv.ParseInt(line[7:], 10, 64)
+		case strings.HasPrefix(line, statSizePack):
+			number, _ := strconv.ParseInt(line[11:], 10, 64)
+			repoSize.SizePack = number * 1024
+		case strings.HasPrefix(line, statPrunePackage):
+			repoSize.PrunePack, _ = strconv.ParseInt(line[16:], 10, 64)
+		case strings.HasPrefix(line, statGarbage):
+			repoSize.Garbage, _ = strconv.ParseInt(line[9:], 10, 64)
+		case strings.HasPrefix(line, statSizeGarbage):
+			number, _ := strconv.ParseInt(line[14:], 10, 64)
+			repoSize.SizeGarbage = number * 1024
+		}
+	}
+	return repoSize
+}
+
 // GetLatestCommitTime returns time for latest commit in repository (across all branches)
 func GetLatestCommitTime(ctx context.Context, repoPath string) (time.Time, error) {
 	cmd := NewCommand(ctx, "for-each-ref", "--sort=-committerdate", BranchPrefix, "--count", "1", "--format=%(committerdate)")

modules/repository/create.go

@@ -173,6 +173,7 @@ type CreateRepoOptions struct {
 	Status         repo_model.RepositoryStatus
 	TrustModel     repo_model.TrustModelType
 	MirrorInterval string
+	SizeLimit      int64
 }
 
 // CreateRepository creates a repository for the user/organization.
@@ -210,6 +211,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m
 		Status:                          opts.Status,
 		IsEmpty:                         !opts.AutoInit,
 		TrustModel:                      opts.TrustModel,
+		SizeLimit:                       opts.SizeLimit,
 		IsMirror:                        opts.IsMirror,
 		DefaultBranch:                   opts.DefaultBranch,
 	}

modules/setting/repository.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
+
+	"github.com/dustin/go-humanize"
 )
 
 // enumerates all the policy repository creating
@@ -269,12 +271,35 @@ var (
 	RepoArchive = struct {
 		Storage
 	}{}
+
+	EnableSizeLimit = true
+	RepoSizeLimit   int64
 )
 
+func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) error {
+	EnableSizeLimit = enableSizeLimit
+	RepoSizeLimit = repoSizeLimit
+	sec := CfgProvider.Section("repository")
+	if EnableSizeLimit {
+		sec.Key("ENABLE_SIZE_LIMIT").SetValue("true")
+	} else {
+		sec.Key("ENABLE_SIZE_LIMIT").SetValue("false")
+	}
+
+	sec.Key("REPO_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(RepoSizeLimit)))
+	return nil
+}
+
 func loadRepositoryFrom(rootCfg ConfigProvider) {
 	var err error
+
 	// Determine and create root git repository path.
 	sec := rootCfg.Section("repository")
+	EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool(false)
+
+	v, _ := humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0"))
+	RepoSizeLimit = int64(v)
+
 	Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool()
 	Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool()
 	Repository.GoGetCloneURLProtocol = sec.Key("GO_GET_CLONE_URL_PROTOCOL").MustString("https")

modules/structs/repo.go

@@ -138,6 +138,8 @@ type CreateRepoOption struct {
 	// TrustModel of the repository
 	// enum: default,collaborator,committer,collaboratorcommitter
 	TrustModel string `json:"trust_model"`
+	// SizeLimit of the repository
+	SizeLimit int64 `json:"size_limit"`
 }
 
 // EditRepoOption options when editing a repository's properties
@@ -202,6 +204,8 @@ type EditRepoOption struct {
 	DefaultAllowMaintainerEdit *bool `json:"default_allow_maintainer_edit,omitempty"`
 	// set to `true` to archive this repository.
 	Archived *bool `json:"archived,omitempty"`
+	// SizeLimit of the repository.
+	SizeLimit *int64 `json:"size_limit,omitempty"`
 	// set to a string like `8h30m0s` to set the mirror interval time
 	MirrorInterval *string `json:"mirror_interval,omitempty"`
 	// enable prune - remove obsolete remote-tracking references

options/locale/locale_en-US.ini

@@ -910,6 +910,7 @@ owner_helper = Some organizations may not show up in the dropdown due to a maxim
 repo_name = Repository Name
 repo_name_helper = Good repository names use short, memorable and unique keywords.
 repo_size = Repository Size
+repo_size_limit = Repository Size Limit
 template = Template
 template_select = Select a template.
 template_helper = Make repository a template
@@ -1025,6 +1026,8 @@ form.reach_limit_of_creation_1 = The owner has already reached the limit of %d r
 form.reach_limit_of_creation_n = The owner has already reached the limit of %d repositories.
 form.name_reserved = The repository name "%s" is reserved.
 form.name_pattern_not_allowed = The pattern "%s" is not allowed in a repository name.
+form.repo_size_limit_negative = Repository size limitation cannot be negative.
+form.repo_size_limit_only_by_admins = Only administrators can change the repository size limitation.
 
 need_auth = Authorization
 migrate_options = Migration Options
@@ -3043,6 +3046,13 @@ config.picture_service = Picture Service
 config.disable_gravatar = Disable Gravatar
 config.enable_federated_avatar = Enable Federated Avatars
 
+config.repository_config = Repository Configuration
+config.enable_size_limit = Enable Size Limit
+config.repo_size_limit = Default Repository Size Limit
+config.invalid_repo_size = Invalid repository size.
+config.save_repo_size_setting_failed = Failed to save global repository settings
+config.repository_setting_success = Global repository setting has been updated
+
 config.git_config = Git Configuration
 config.git_disable_diff_highlight = Disable Diff Syntax Highlight
 config.git_max_diff_lines = Max Diff Lines (for a single file)

routers/api/v1/repo/repo.go

@@ -252,6 +252,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *user_model.User, opt api.Cre
 		DefaultBranch: opt.DefaultBranch,
 		TrustModel:    repo_model.ToTrustModel(opt.TrustModel),
 		IsTemplate:    opt.Template,
+		SizeLimit:     opt.SizeLimit,
 	})
 	if err != nil {
 		if repo_model.IsErrRepoAlreadyExist(err) {
@@ -728,6 +729,10 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
 		repo.DefaultBranch = *opts.DefaultBranch
 	}
 
+	if opts.SizeLimit != nil {
+		repo.SizeLimit = *opts.SizeLimit
+	}
+
 	if err := repo_service.UpdateRepository(ctx, repo, visibilityChanged); err != nil {
 		ctx.Error(http.StatusInternalServerError, "UpdateRepository", err)
 		return err