Add default storage configurations (#12813)

Signed-off-by: Andrew Thornton <[email protected]>
Co-authored-by: zeripath <[email protected]>

.golangci.yml

@@ -98,3 +98,6 @@ issues:
     - path: models/update.go
       linters:
         - unused
+    - path: cmd/dump.go
+      linters:
+        - dupl

cmd/dump.go

@@ -18,6 +18,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/util"
 
 	"gitea.com/macaron/session"
@@ -57,6 +58,8 @@ func addRecursive(w archiver.Writer, dirPath string, absPath string, verbose boo
 	if err != nil {
 		return fmt.Errorf("Could not open directory %s: %s", absPath, err)
 	}
+	defer dir.Close()
+
 	files, err := dir.Readdir(0)
 	if err != nil {
 		return fmt.Errorf("Unable to list files in %s: %s", absPath, err)
@@ -197,6 +200,10 @@ func runDump(ctx *cli.Context) error {
 		return err
 	}
 
+	if err := storage.Init(); err != nil {
+		return err
+	}
+
 	if file == nil {
 		file, err = os.Create(fileName)
 		if err != nil {
@@ -231,11 +238,21 @@ func runDump(ctx *cli.Context) error {
 			fatal("Failed to include repositories: %v", err)
 		}
 
-		if _, err := os.Stat(setting.LFS.ContentPath); !os.IsNotExist(err) {
-			log.Info("Dumping lfs... %s", setting.LFS.ContentPath)
-			if err := addRecursive(w, "lfs", setting.LFS.ContentPath, verbose); err != nil {
-				fatal("Failed to include lfs: %v", err)
+		if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error {
+			info, err := object.Stat()
+			if err != nil {
+				return err
 			}
+
+			return w.Write(archiver.File{
+				FileInfo: archiver.FileInfo{
+					FileInfo:   info,
+					CustomName: path.Join("data", "lfs", objPath),
+				},
+				ReadCloser: object,
+			})
+		}); err != nil {
+			fatal("Failed to dump LFS objects: %v", err)
 		}
 	}
 
@@ -302,13 +319,31 @@ func runDump(ctx *cli.Context) error {
 		}
 
 		excludes = append(excludes, setting.RepoRootPath)
-		excludes = append(excludes, setting.LFS.ContentPath)
+		excludes = append(excludes, setting.LFS.Path)
+		excludes = append(excludes, setting.Attachment.Path)
 		excludes = append(excludes, setting.LogRootPath)
 		if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
 			fatal("Failed to include data directory: %v", err)
 		}
 	}
 
+	if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error {
+		info, err := object.Stat()
+		if err != nil {
+			return err
+		}
+
+		return w.Write(archiver.File{
+			FileInfo: archiver.FileInfo{
+				FileInfo:   info,
+				CustomName: path.Join("data", "attachments", objPath),
+			},
+			ReadCloser: object,
+		})
+	}); err != nil {
+		fatal("Failed to dump attachments: %v", err)
+	}
+
 	// Doesn't check if LogRootPath exists before processing --skip-log intentionally,
 	// ensuring that it's clear the dump is skipped whether the directory's initialized
 	// yet or not.

cmd/migrate_storage.go

@@ -7,6 +7,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/migrations"
@@ -30,8 +31,8 @@ var CmdMigrateStorage = cli.Command{
 			Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
 		},
 		cli.StringFlag{
-			Name:  "store, s",
-			Value: "local",
+			Name:  "storage, s",
+			Value: setting.LocalStorageType,
 			Usage: "New storage type, local or minio",
 		},
 		cli.StringFlag{
@@ -112,15 +113,15 @@ func runMigrateStorage(ctx *cli.Context) error {
 
 	var dstStorage storage.ObjectStorage
 	var err error
-	switch ctx.String("store") {
-	case "local":
+	switch strings.ToLower(ctx.String("storage")) {
+	case setting.LocalStorageType:
 		p := ctx.String("path")
 		if p == "" {
-			log.Fatal("Path must be given when store is loal")
+			log.Fatal("Path must be given when storage is loal")
 			return nil
 		}
 		dstStorage, err = storage.NewLocalStorage(p)
-	case "minio":
+	case setting.MinioStorageType:
 		dstStorage, err = storage.NewMinioStorage(
 			context.Background(),
 			ctx.String("minio-endpoint"),
@@ -132,14 +133,14 @@ func runMigrateStorage(ctx *cli.Context) error {
 			ctx.Bool("minio-use-ssl"),
 		)
 	default:
-		return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
+		return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage"))
 	}
 
 	if err != nil {
 		return err
 	}
 
-	tp := ctx.String("type")
+	tp := strings.ToLower(ctx.String("type"))
 	switch tp {
 	case "attachments":
 		if err := migrateAttachments(dstStorage); err != nil {

custom/conf/app.example.ini

@@ -778,25 +778,25 @@ MAX_SIZE = 4
 MAX_FILES = 5
 ; Storage type for attachments, `local` for local disk or `minio` for s3 compatible
 ; object storage service, default is `local`.
-STORE_TYPE = local
+STORAGE_TYPE = local
 ; Allows the storage driver to redirect to authenticated URLs to serve files directly
 ; Currently, only `minio` is supported.
 SERVE_DIRECT = false
-; Path for attachments. Defaults to `data/attachments` only available when STORE_TYPE is `local`
+; Path for attachments. Defaults to `data/attachments` only available when STORAGE_TYPE is `local`
 PATH = data/attachments
-; Minio endpoint to connect only available when STORE_TYPE is `minio`
+; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
 MINIO_ENDPOINT = localhost:9000
-; Minio accessKeyID to connect only available when STORE_TYPE is `minio`
+; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
 MINIO_ACCESS_KEY_ID =
-; Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
+; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
 MINIO_SECRET_ACCESS_KEY =
-; Minio bucket to store the attachments only available when STORE_TYPE is `minio`
+; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
 MINIO_BUCKET = gitea
-; Minio location to create bucket only available when STORE_TYPE is `minio`
+; Minio location to create bucket only available when STORAGE_TYPE is `minio`
 MINIO_LOCATION = us-east-1
-; Minio base path on the bucket only available when STORE_TYPE is `minio`
+; Minio base path on the bucket only available when STORAGE_TYPE is `minio`
 MINIO_BASE_PATH = attachments/
-; Minio enabled ssl only available when STORE_TYPE is `minio`
+; Minio enabled ssl only available when STORAGE_TYPE is `minio`
 MINIO_USE_SSL = false
 
 [time]
@@ -1161,3 +1161,28 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
 MAX_ATTEMPTS = 3
 ; Backoff time per http/https request retry (seconds)
 RETRY_BACKOFF = 3
+
+; default storage for attachments, lfs and avatars
+[storage]
+; storage type
+STORAGE_TYPE = local
+
+; lfs storage will override storage
+[lfs]
+STORAGE_TYPE = local
+
+; customize storage
+;[storage.my_minio]
+;STORAGE_TYPE = minio
+; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
+;MINIO_ENDPOINT = localhost:9000
+; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+;MINIO_ACCESS_KEY_ID =
+; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
+;MINIO_SECRET_ACCESS_KEY =
+; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
+;MINIO_BUCKET = gitea
+; Minio location to create bucket only available when STORAGE_TYPE is `minio`
+;MINIO_LOCATION = us-east-1
+; Minio enabled ssl only available when STORAGE_TYPE is `minio`
+;MINIO_USE_SSL = false

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -220,16 +220,6 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
 
 - `LFS_START_SERVER`: **false**: Enables git-lfs support.
-- `LFS_STORE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service.
-- `LFS_SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
-- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `LFS_STORE_TYPE` is `local`.
-- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `LFS_STORE_TYPE` is `minio`
-- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `LFS_STORE_TYPE` is `minio`
-- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `LFS_STORE_TYPE is` `minio`
-- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `LFS_STORE_TYPE` is `minio`
-- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `LFS_STORE_TYPE` is `minio`
-- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `LFS_STORE_TYPE` is `minio`
-- `LFS_MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `LFS_STORE_TYPE` is `minio`
 - `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
 - `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
 - `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
@@ -501,16 +491,16 @@ relation to port exhaustion.
    Use `*/*` for all types.
 - `MAX_SIZE`: **4**: Maximum size (MB).
 - `MAX_FILES`: **5**: Maximum number of attachments that can be uploaded at once.
-- `STORE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local`.
+- `STORAGE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`
 - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
-- `PATH`: **data/attachments**: Path to store attachments only available when STORE_TYPE is `local`
-- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORE_TYPE is `minio`
-- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORE_TYPE is `minio`
-- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
-- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORE_TYPE is `minio`
-- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORE_TYPE is `minio`
-- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORE_TYPE is `minio`
-- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORE_TYPE is `minio`
+- `PATH`: **data/attachments**: Path to store attachments only available when STORAGE_TYPE is `local`
+- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORAGE_TYPE is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
+- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
+- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORAGE_TYPE is `minio`
+- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`
+- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio`
 
 ## Log (`log`)
 
@@ -714,6 +704,56 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
 - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
 
+## LFS (`lfs`)
+
+Storage configuration for lfs data. It will be derived from default `[storage]` or
+`[storage.xxx]` when set `STORAGE_TYPE` to `xxx`. When derived, the default of `PATH` 
+is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`.
+
+- `STORAGE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service or other name defined with `[storage.xxx]`
+- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
+- `CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `STORAGE_TYPE` is `local`.
+- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
+- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
+- `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
+- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
+- `MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
+- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
+
+## Storage (`storage`)
+
+Default storage configuration for attachments, lfs, avatars and etc.
+
+- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
+- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
+- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
+- `MINIO_BUCKET`: **gitea**: Minio bucket to store the data only available when `STORAGE_TYPE` is `minio`
+- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
+- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
+
+And you can also define a customize storage like below:
+
+```ini
+[storage.my_minio]
+STORAGE_TYPE = minio
+; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
+MINIO_ENDPOINT = localhost:9000
+; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
+MINIO_ACCESS_KEY_ID =
+; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
+MINIO_SECRET_ACCESS_KEY =
+; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
+MINIO_BUCKET = gitea
+; Minio location to create bucket only available when STORAGE_TYPE is `minio`
+MINIO_LOCATION = us-east-1
+; Minio enabled ssl only available when STORAGE_TYPE is `minio`
+MINIO_USE_SSL = false
+```
+
+And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`.
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.