fix: increase beego max memory and upload size (goharbor#19578)

1. Increase the default beego max memory and upload size from 32GB to 128GB. 2. Support customize the two beego configs from env. Signed-off-by: chlins <[email protected]> Co-authored-by: Wang Yan <[email protected]>
stonezdj · Nov 21, 2023 · 553c85e · 553c85e
1 parent 996e57b
commit 553c85e
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 3 deletions.
diff --git a/src/common/const.go b/src/common/const.go
@@ -231,4 +231,14 @@ const (
 	QuotaUpdateProvider = "quota_update_provider"
 	// IllegalCharsInUsername is the illegal chars in username
 	IllegalCharsInUsername = `,"~#%$`
+
+	// Beego web config
+	// BeegoMaxMemoryBytes is the max memory(bytes) of the beego web config
+	BeegoMaxMemoryBytes = "beego_max_memory_bytes"
+	// DefaultBeegoMaxMemoryBytes sets default max memory to 128GB
+	DefaultBeegoMaxMemoryBytes = 1 << 37
+	// BeegoMaxUploadSizeBytes is the max upload size(bytes) of the beego web config
+	BeegoMaxUploadSizeBytes = "beego_max_upload_size_bytes"
+	// DefaultBeegoMaxUploadSizeBytes sets default max upload size to 128GB
+	DefaultBeegoMaxUploadSizeBytes = 1 << 37
 )
diff --git a/src/core/main.go b/src/core/main.go
@@ -127,8 +127,6 @@ func main() {
 
 	web.BConfig.WebConfig.Session.SessionOn = true
 	web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
-	web.BConfig.MaxMemory = 1 << 35     // (32GB)
-	web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
 	// the core db used for beego session
 	redisCoreURL := os.Getenv("_REDIS_URL_CORE")
 	if len(redisCoreURL) > 0 {
@@ -163,6 +161,12 @@ func main() {
 	log.Info("initializing configurations...")
 	config.Init()
 	log.Info("configurations initialization completed")
+
+	// default beego max memory and max upload size is 128GB, consider from some AI related image would be large,
+	// also support customize it from the environment variables if the default value cannot satisfy some scenarios.
+	web.BConfig.MaxMemory = config.GetBeegoMaxMemoryBytes()
+	web.BConfig.MaxUploadSize = config.GetBeegoMaxUploadSizeBytes()
+
 	metricCfg := config.Metric()
 	if metricCfg.Enabled {
 		metric.RegisterCollectors()

diff --git a/src/lib/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go
@@ -14,7 +14,11 @@
 
 package metadata
 
-import "github.com/goharbor/harbor/src/common"
+import (
+	"fmt"
+
+	"github.com/goharbor/harbor/src/common"
+)
 
 // Item - Configure item include default value, type, env name
 type Item struct {
@@ -193,5 +197,8 @@ var (
 
 		{Name: common.BannerMessage, Scope: UserScope, Group: BasicGroup, EnvKey: "BANNER_MESSAGE", DefaultValue: "", ItemType: &StringType{}, Editable: true, Description: `The customized banner message for the UI`},
 		{Name: common.QuotaUpdateProvider, Scope: SystemScope, Group: BasicGroup, EnvKey: "QUOTA_UPDATE_PROVIDER", DefaultValue: "db", ItemType: &StringType{}, Editable: false, Description: `The provider for updating quota, 'db' or 'redis' is supported`},
+
+		{Name: common.BeegoMaxMemoryBytes, Scope: SystemScope, Group: BasicGroup, EnvKey: "BEEGO_MAX_MEMORY_BYTES", DefaultValue: fmt.Sprintf("%d", common.DefaultBeegoMaxMemoryBytes), ItemType: &Int64Type{}, Editable: false, Description: `The bytes for limiting the beego max memory, default is 128GB`},
+		{Name: common.BeegoMaxUploadSizeBytes, Scope: SystemScope, Group: BasicGroup, EnvKey: "BEEGO_MAX_UPLOAD_SIZE_BYTES", DefaultValue: fmt.Sprintf("%d", common.DefaultBeegoMaxUploadSizeBytes), ItemType: &Int64Type{}, Editable: false, Description: `The bytes for limiting the beego max upload size, default it 128GB`},
 	}
 )
diff --git a/src/lib/config/systemconfig.go b/src/lib/config/systemconfig.go
@@ -132,6 +132,16 @@ func GetQuotaUpdateProvider() string {
 	return DefaultMgr().Get(backgroundCtx, common.QuotaUpdateProvider).GetString()
 }
 
+// GetBeegoMaxMemoryBytes returns the max memory bytes of beego config
+func GetBeegoMaxMemoryBytes() int64 {
+	return DefaultMgr().Get(backgroundCtx, common.BeegoMaxMemoryBytes).GetInt64()
+}
+
+// GetBeegoMaxUploadSizeBytes returns the max upload size bytes of beego config
+func GetBeegoMaxUploadSizeBytes() int64 {
+	return DefaultMgr().Get(backgroundCtx, common.BeegoMaxUploadSizeBytes).GetInt64()
+}
+
 // WithTrivy returns a bool value to indicate if Harbor's deployed with Trivy.
 func WithTrivy() bool {
 	return DefaultMgr().Get(backgroundCtx, common.WithTrivy).GetBool()