From 8aada1849f949d0c7f023f11061cacc957e0df8f Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 11 Apr 2023 23:42:22 -0400 Subject: [PATCH 01/12] update BSDmakefile to latest version from upstream (#24063) from https://github.com/neosmart/gmake-proxy --- BSDmakefile | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/BSDmakefile b/BSDmakefile index 2b4cae678b519..13174f8fd22b4 100644 --- a/BSDmakefile +++ b/BSDmakefile @@ -1,6 +1,7 @@ # GNU makefile proxy script for BSD make +# # Written and maintained by Mahmoud Al-Qudsi -# Copyright NeoSmart Technologies 2014-2018 +# Copyright NeoSmart Technologies 2014-2019 # Obtain updates from # # Redistribution and use in source and binary forms, with or without @@ -26,26 +27,32 @@ JARG = GMAKE = "gmake" -#When gmake is called from another make instance, -w is automatically added -#which causes extraneous messages about directory changes to be emitted. -#--no-print-directory silences these messages. +# When gmake is called from another make instance, -w is automatically added +# which causes extraneous messages about directory changes to be emitted. +# Running with --no-print-directory silences these messages. GARGS = "--no-print-directory" .if "$(.MAKE.JOBS)" != "" -JARG = -j$(.MAKE.JOBS) + JARG = -j$(.MAKE.JOBS) .endif -#by default bmake will cd into ./obj first +# bmake prefers out-of-source builds and tries to cd into ./obj (among others) +# where possible. GNU Make doesn't, so override that value. .OBJDIR: ./ +# The GNU convention is to use the lowercased `prefix` variable/macro to +# specify the installation directory. Humor them. +GPREFIX = "" +.if defined(PREFIX) && ! defined(prefix) + GPREFIX = 'prefix = "$(PREFIX)"' +.endif + +.BEGIN: .SILENT + which $(GMAKE) || printf "Error: GNU Make is required!\n\n" 1>&2 && false + .PHONY: FRC $(.TARGETS): FRC - $(GMAKE) $(GARGS) $(.TARGETS:S,.DONE,,) $(JARG) + $(GMAKE) $(GPREFIX) $(GARGS) $(.TARGETS:S,.DONE,,) $(JARG) .DONE .DEFAULT: .SILENT - $(GMAKE) $(GARGS) $(.TARGETS:S,.DONE,,) $(JARG) - -.ERROR: .SILENT - if ! which $(GMAKE) > /dev/null; then \ - echo "GNU Make is required!"; \ - fi + $(GMAKE) $(GPREFIX) $(GARGS) $(.TARGETS:S,.DONE,,) $(JARG) From 97176754beb4de23fa0f68df715c4737919c93b0 Mon Sep 17 00:00:00 2001 From: Yarden Shoham Date: Wed, 12 Apr 2023 09:29:49 +0300 Subject: [PATCH 02/12] Localize milestone related time strings (#24051) - With #23988 in place, we can improve these timestamps --------- Co-authored-by: silverwind --- models/issues/milestone.go | 5 ----- options/locale/locale_en-US.ini | 2 +- templates/repo/issue/milestone_issues.tmpl | 2 +- templates/repo/issue/milestones.tmpl | 4 ++-- templates/user/dashboard/milestones.tmpl | 2 +- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/models/issues/milestone.go b/models/issues/milestone.go index 8255db38f9137..ffe5c8eb509ba 100644 --- a/models/issues/milestone.go +++ b/models/issues/milestone.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "strings" - "time" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" @@ -65,7 +64,6 @@ type Milestone struct { DeadlineString string `xorm:"-"` TotalTrackedTime int64 `xorm:"-"` - TimeSinceUpdate int64 `xorm:"-"` } func init() { @@ -84,9 +82,6 @@ func (m *Milestone) BeforeUpdate() { // AfterLoad is invoked from XORM after setting the value of a field of // this object. func (m *Milestone) AfterLoad() { - if !m.UpdatedUnix.IsZero() { - m.TimeSinceUpdate = time.Now().Unix() - m.UpdatedUnix.AsTime().Unix() - } m.NumOpenIssues = m.NumIssues - m.NumClosedIssues if m.DeadlineUnix.Year() == 9999 { return diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0a10b70d9daa1..cf3208b5bdcb1 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1710,7 +1710,7 @@ pulls.delete.text = Do you really want to delete this pull request? (This will p milestones.new = New Milestone milestones.closed = Closed %s -milestones.update_ago = Updated %s ago +milestones.updated = Updated milestones.no_due_date = No due date milestones.open = Open milestones.close = Close diff --git a/templates/repo/issue/milestone_issues.tmpl b/templates/repo/issue/milestone_issues.tmpl index c3c25843e8e19..1d55eb39cdc11 100644 --- a/templates/repo/issue/milestone_issues.tmpl +++ b/templates/repo/issue/milestone_issues.tmpl @@ -35,7 +35,7 @@ {{else}} {{svg "octicon-calendar"}} {{if .Milestone.DeadlineString}} - {{.Milestone.DeadlineString}} + {{template "shared/datetime/short" (dict "Datetime" .Milestone.DeadlineString "Fallback" .Milestone.DeadlineString)}} {{else}} {{$.locale.Tr "repo.milestones.no_due_date"}} {{end}} diff --git a/templates/repo/issue/milestones.tmpl b/templates/repo/issue/milestones.tmpl index 42a6c4f91955f..2864d38a08e9b 100644 --- a/templates/repo/issue/milestones.tmpl +++ b/templates/repo/issue/milestones.tmpl @@ -77,7 +77,7 @@ {{else}} {{svg "octicon-calendar"}} {{if .DeadlineString}} - {{.DeadlineString}} + {{template "shared/datetime/short" (dict "Datetime" .DeadlineString "Fallback" .DeadlineString)}} {{else}} {{$.locale.Tr "repo.milestones.no_due_date"}} {{end}} @@ -88,7 +88,7 @@ {{svg "octicon-check" 16 "gt-mr-3"}} {{LocaleNumber .NumClosedIssues}} {{$.locale.Tr "repo.issues.closed_title"}} {{if .TotalTrackedTime}}{{svg "octicon-clock"}} {{.TotalTrackedTime|Sec2Time}}{{end}} - {{if .UpdatedUnix}}{{svg "octicon-clock"}} {{$.locale.Tr "repo.milestones.update_ago" (.TimeSinceUpdate|Sec2Time)}}{{end}} + {{if .UpdatedUnix}}{{svg "octicon-clock"}} {{$.locale.Tr "repo.milestones.updated"}} {{TimeSinceUnix .UpdatedUnix $.locale}}{{end}} {{if and (or $.CanWriteIssues $.CanWritePulls) (not $.Repository.IsArchived)}} diff --git a/templates/user/dashboard/milestones.tmpl b/templates/user/dashboard/milestones.tmpl index 6788aa09a2ac8..99151597217ae 100644 --- a/templates/user/dashboard/milestones.tmpl +++ b/templates/user/dashboard/milestones.tmpl @@ -97,7 +97,7 @@ {{else}} {{svg "octicon-calendar"}} {{if .DeadlineString}} - {{.DeadlineString}} + {{template "shared/datetime/short" (dict "Datetime" .DeadlineString "Fallback" .DeadlineString)}} {{else}} {{$.locale.Tr "repo.milestones.no_due_date"}} {{end}} From e03e827dcb27a4cd34dd4f9da96ec8d15aaa5c5a Mon Sep 17 00:00:00 2001 From: sillyguodong <33891828+sillyguodong@users.noreply.github.com> Date: Wed, 12 Apr 2023 15:06:39 +0800 Subject: [PATCH 03/12] Expand selected file when clicking file tree (#24041) Auto expand the selected file when clicking the file item of the file tree. This is consistent with Github's behavior. https://user-images.githubusercontent.com/33891828/231048124-61f180af-adba-42d7-9ffa-626e1de04aed.mov --- web_src/js/components/DiffFileTree.vue | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/web_src/js/components/DiffFileTree.vue b/web_src/js/components/DiffFileTree.vue index 9fc08af1fc2f1..c5a62dd4cc669 100644 --- a/web_src/js/components/DiffFileTree.vue +++ b/web_src/js/components/DiffFileTree.vue @@ -18,6 +18,7 @@ import DiffFileTreeItem from './DiffFileTreeItem.vue'; import {doLoadMoreFiles} from '../features/repo-diff.js'; import {toggleElem} from '../utils/dom.js'; import {DiffTreeStore} from '../modules/stores.js'; +import {setFileFolding} from '../features/file-fold.js'; const {pageData} = window.config; const LOCAL_STORAGE_KEY = 'diff_file_tree_visible'; @@ -104,6 +105,7 @@ export default { this.hashChangeListener = () => { this.store.selectedItem = window.location.hash; + this.expandSelectedFile(); }; this.hashChangeListener(); window.addEventListener('hashchange', this.hashChangeListener); @@ -113,6 +115,14 @@ export default { window.removeEventListener('hashchange', this.hashChangeListener); }, methods: { + expandSelectedFile() { + // expand file if the selected file is folded + if (this.store.selectedItem) { + const box = document.querySelector(this.store.selectedItem); + const folded = box?.getAttribute('data-folded') === 'true'; + if (folded) setFileFolding(box, box.querySelector('.fold-file'), false); + } + }, toggleVisibility() { this.updateVisibility(!this.fileTreeIsVisible); }, From 42919ccb7cd32ab67d0878baf2bac6cd007899a8 Mon Sep 17 00:00:00 2001 From: JakobDev Date: Wed, 12 Apr 2023 11:05:23 +0200 Subject: [PATCH 04/12] Make Release Download URLs predictable (#23891) As promised in #23817, I have this made a PR to make Release Download URLs predictable. It currently follows the schema `/releases/download//`. this already works, but it is nowhere shown in the UI or the API. The Problem is, that it is currently possible to have multiple files with the same name (why do we even allow this) for a release. I had written some Code to check, if a Release has 2 or more files with the same Name. If yes, it uses the old `attachments/` URlL if no it uses the new fancy URL. I had also changed `/releases/download//` to directly serve the File instead of redirecting, so people who who use automatic update checker don't end up with the `attachments/` URL. Fixes #10919 --------- Co-authored-by: a1012112796 <1012112796@qq.com> --- models/repo/attachment.go | 27 ++++++++++++++++----------- models/repo/release.go | 29 +++++++++++++++++++++++++++++ routers/web/repo/attachment.go | 11 ++++++++--- routers/web/repo/release.go | 6 ++++++ routers/web/repo/repo.go | 2 +- 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/models/repo/attachment.go b/models/repo/attachment.go index cb05386d93871..93b83aae8a8db 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -18,17 +18,18 @@ import ( // Attachment represent a attachment of issue/comment/release. type Attachment struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - RepoID int64 `xorm:"INDEX"` // this should not be zero - IssueID int64 `xorm:"INDEX"` // maybe zero when creating - ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating - UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 - Name string - DownloadCount int64 `xorm:"DEFAULT 0"` - Size int64 `xorm:"DEFAULT 0"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + RepoID int64 `xorm:"INDEX"` // this should not be zero + IssueID int64 `xorm:"INDEX"` // maybe zero when creating + ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 + Name string + DownloadCount int64 `xorm:"DEFAULT 0"` + Size int64 `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + CustomDownloadURL string `xorm:"-"` } func init() { @@ -57,6 +58,10 @@ func (a *Attachment) RelativePath() string { // DownloadURL returns the download url of the attached file func (a *Attachment) DownloadURL() string { + if a.CustomDownloadURL != "" { + return a.CustomDownloadURL + } + return setting.AppURL + "attachments/" + url.PathEscape(a.UUID) } diff --git a/models/repo/release.go b/models/repo/release.go index c8dd7fbc7a368..75eb27f07437b 100644 --- a/models/repo/release.go +++ b/models/repo/release.go @@ -7,6 +7,7 @@ package repo import ( "context" "fmt" + "net/url" "sort" "strconv" "strings" @@ -372,6 +373,34 @@ func GetReleaseAttachments(ctx context.Context, rels ...*Release) (err error) { sortedRels.Rel[currentIndex].Attachments = append(sortedRels.Rel[currentIndex].Attachments, attachment) } + // Makes URL's predictable + for _, release := range rels { + // If we have no Repo, we don't need to execute this loop + if release.Repo == nil { + continue + } + + // Check if there are two or more attachments with the same name + hasDuplicates := false + foundNames := make(map[string]bool) + for _, attachment := range release.Attachments { + _, found := foundNames[attachment.Name] + if found { + hasDuplicates = true + break + } else { + foundNames[attachment.Name] = true + } + } + + // If the names unique, use the URL with the Name instead of the UUID + if !hasDuplicates { + for _, attachment := range release.Attachments { + attachment.CustomDownloadURL = release.Repo.HTMLURL() + "/releases/download/" + url.PathEscape(release.TagName) + "/" + url.PathEscape(attachment.Name) + } + } + } + return err } diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index c6d8828fac603..9fb9cb00bf90c 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -86,9 +86,9 @@ func DeleteAttachment(ctx *context.Context) { }) } -// GetAttachment serve attachments -func GetAttachment(ctx *context.Context) { - attach, err := repo_model.GetAttachmentByUUID(ctx, ctx.Params(":uuid")) +// GetAttachment serve attachments with the given UUID +func ServeAttachment(ctx *context.Context, uuid string) { + attach, err := repo_model.GetAttachmentByUUID(ctx, uuid) if err != nil { if repo_model.IsErrAttachmentNotExist(err) { ctx.Error(http.StatusNotFound) @@ -153,3 +153,8 @@ func GetAttachment(ctx *context.Context) { return } } + +// GetAttachment serve attachments +func GetAttachment(ctx *context.Context) { + ServeAttachment(ctx, ctx.Params(":uuid")) +} diff --git a/routers/web/repo/release.go b/routers/web/repo/release.go index 14ef1372c0404..e8caa2cbb755e 100644 --- a/routers/web/repo/release.go +++ b/routers/web/repo/release.go @@ -142,6 +142,10 @@ func releasesOrTags(ctx *context.Context, isTagList bool) { return } + for _, release := range releases { + release.Repo = ctx.Repo.Repository + } + if err = repo_model.GetReleaseAttachments(ctx, releases...); err != nil { ctx.ServerError("GetReleaseAttachments", err) return @@ -248,6 +252,8 @@ func SingleRelease(ctx *context.Context) { ctx.Data["Title"] = release.Title } + release.Repo = ctx.Repo.Repository + err = repo_model.GetReleaseAttachments(ctx, release) if err != nil { ctx.ServerError("GetReleaseAttachments", err) diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 9b80e85324769..5a97c5190c275 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -373,7 +373,7 @@ func RedirectDownload(ctx *context.Context) { return } if att != nil { - ctx.Redirect(att.DownloadURL()) + ServeAttachment(ctx, att.UUID) return } } From 50a72e7a83a16d183a264e969a73cdbc7fb808f4 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 12 Apr 2023 18:16:45 +0800 Subject: [PATCH 05/12] Use a general approach to access custom/static/builtin assets (#24022) The idea is to use a Layered Asset File-system (modules/assetfs/layered.go) For example: when there are 2 layers: "custom", "builtin", when access to asset "my/page.tmpl", the Layered Asset File-system will first try to use "custom" assets, if not found, then use "builtin" assets. This approach will hugely simplify a lot of code, make them testable. Other changes: * Simplify the AssetsHandlerFunc code * Simplify the `gitea embedded` sub-command code --------- Co-authored-by: Jason Song Co-authored-by: Lunny Xiao --- cmd/embedded.go | 122 ++++++-------- cmd/embedded_stub.go | 29 ---- modules/assetfs/layered.go | 260 +++++++++++++++++++++++++++++ modules/assetfs/layered_test.go | 109 ++++++++++++ modules/auth/pam/pam_stub.go | 6 +- modules/context/context.go | 12 +- modules/doctor/paths.go | 3 +- modules/options/base.go | 118 ++----------- modules/options/dynamic.go | 25 +-- modules/options/options.go | 45 ----- modules/options/static.go | 95 +---------- modules/public/public.go | 92 +++++----- modules/public/serve_dynamic.go | 15 +- modules/public/serve_static.go | 66 +------- modules/repository/init.go | 2 +- modules/setting/asset_dynamic.go | 8 + modules/setting/asset_static.go | 8 + modules/svg/discover_bindata.go | 30 ---- modules/svg/discover_nobindata.go | 29 ---- modules/svg/svg.go | 29 +++- modules/templates/base.go | 93 +++-------- modules/templates/dynamic.go | 72 +------- modules/templates/htmlrenderer.go | 62 ++----- modules/templates/mailer.go | 63 ++----- modules/templates/static.go | 103 +----------- modules/translation/translation.go | 14 +- modules/util/path.go | 19 +-- modules/util/path_test.go | 2 +- modules/util/timer.go | 28 ++++ modules/util/timer_test.go | 30 ++++ modules/watcher/watcher.go | 114 ------------- routers/init.go | 9 +- routers/install/routes.go | 6 +- routers/install/setting.go | 3 +- routers/web/devtest/devtest.go | 15 +- routers/web/web.go | 6 +- 36 files changed, 688 insertions(+), 1054 deletions(-) delete mode 100644 cmd/embedded_stub.go create mode 100644 modules/assetfs/layered.go create mode 100644 modules/assetfs/layered_test.go delete mode 100644 modules/options/options.go create mode 100644 modules/setting/asset_dynamic.go create mode 100644 modules/setting/asset_static.go delete mode 100644 modules/svg/discover_bindata.go delete mode 100644 modules/svg/discover_nobindata.go create mode 100644 modules/util/timer_test.go delete mode 100644 modules/watcher/watcher.go diff --git a/cmd/embedded.go b/cmd/embedded.go index d87fc0187c70d..cee8928ce08db 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -1,8 +1,6 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build bindata - package cmd import ( @@ -10,9 +8,9 @@ import ( "fmt" "os" "path/filepath" - "sort" "strings" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/public" @@ -89,24 +87,20 @@ var ( }, } - sections map[string]*section - assets []asset + matchedAssetFiles []assetFile ) -type section struct { - Path string - Names func() []string - IsDir func(string) (bool, error) - Asset func(string) ([]byte, error) -} - -type asset struct { - Section *section - Name string - Path string +type assetFile struct { + fs *assetfs.LayeredFS + name string + path string } func initEmbeddedExtractor(c *cli.Context) error { + // FIXME: there is a bug, if the user runs `gitea embedded` with a different user or root, + // The setting.Init (loadRunModeFrom) will fail and do log.Fatal + // But the console logger has been deleted, so nothing is printed, the user sees nothing and Gitea just exits. + // Silence the console logger log.DelNamedLogger("console") log.DelNamedLogger(log.DEFAULT) @@ -115,24 +109,14 @@ func initEmbeddedExtractor(c *cli.Context) error { setting.InitProviderAllowEmpty() setting.LoadCommonSettings() - pats, err := getPatterns(c.Args()) + patterns, err := compileCollectPatterns(c.Args()) if err != nil { return err } - sections := make(map[string]*section, 3) - - sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} - sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} - sections["templates"] = §ion{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset} - for _, sec := range sections { - assets = append(assets, buildAssetList(sec, pats, c)...) - } - - // Sort assets - sort.SliceStable(assets, func(i, j int) bool { - return assets[i].Path < assets[j].Path - }) + collectAssetFilesByPattern(c, patterns, "options", options.BuiltinAssets()) + collectAssetFilesByPattern(c, patterns, "public", public.BuiltinAssets()) + collectAssetFilesByPattern(c, patterns, "templates", templates.BuiltinAssets()) return nil } @@ -166,8 +150,8 @@ func runListDo(c *cli.Context) error { return err } - for _, a := range assets { - fmt.Println(a.Path) + for _, a := range matchedAssetFiles { + fmt.Println(a.path) } return nil @@ -178,19 +162,19 @@ func runViewDo(c *cli.Context) error { return err } - if len(assets) == 0 { - return fmt.Errorf("No files matched the given pattern") - } else if len(assets) > 1 { - return fmt.Errorf("Too many files matched the given pattern; try to be more specific") + if len(matchedAssetFiles) == 0 { + return fmt.Errorf("no files matched the given pattern") + } else if len(matchedAssetFiles) > 1 { + return fmt.Errorf("too many files matched the given pattern, try to be more specific") } - data, err := assets[0].Section.Asset(assets[0].Name) + data, err := matchedAssetFiles[0].fs.ReadFile(matchedAssetFiles[0].name) if err != nil { - return fmt.Errorf("%s: %w", assets[0].Path, err) + return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err) } if _, err = os.Stdout.Write(data); err != nil { - return fmt.Errorf("%s: %w", assets[0].Path, err) + return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err) } return nil @@ -202,7 +186,7 @@ func runExtractDo(c *cli.Context) error { } if len(c.Args()) == 0 { - return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)") + return fmt.Errorf("a list of pattern of files to extract is mandatory (e.g. '**' for all)") } destdir := "." @@ -227,7 +211,7 @@ func runExtractDo(c *cli.Context) error { if err != nil { return fmt.Errorf("%s: %s", destdir, err) } else if !fi.IsDir() { - return fmt.Errorf("%s is not a directory.", destdir) + return fmt.Errorf("destination %q is not a directory", destdir) } fmt.Printf("Extracting to %s:\n", destdir) @@ -235,23 +219,23 @@ func runExtractDo(c *cli.Context) error { overwrite := c.Bool("overwrite") rename := c.Bool("rename") - for _, a := range assets { + for _, a := range matchedAssetFiles { if err := extractAsset(destdir, a, overwrite, rename); err != nil { // Non-fatal error - fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) + fmt.Fprintf(os.Stderr, "%s: %v", a.path, err) } } return nil } -func extractAsset(d string, a asset, overwrite, rename bool) error { - dest := filepath.Join(d, filepath.FromSlash(a.Path)) +func extractAsset(d string, a assetFile, overwrite, rename bool) error { + dest := filepath.Join(d, filepath.FromSlash(a.path)) dir := filepath.Dir(dest) - data, err := a.Section.Asset(a.Name) + data, err := a.fs.ReadFile(a.name) if err != nil { - return fmt.Errorf("%s: %w", a.Path, err) + return fmt.Errorf("%s: %w", a.path, err) } if err := os.MkdirAll(dir, os.ModePerm); err != nil { @@ -272,7 +256,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { return fmt.Errorf("%s already exists, but it's not a regular file", dest) } else if rename { if err := util.Rename(dest, dest+".bak"); err != nil { - return fmt.Errorf("Error creating backup for %s: %w", dest, err) + return fmt.Errorf("error creating backup for %s: %w", dest, err) } // Attempt to respect file permissions mask (even if user:group will be set anew) perms = fi.Mode() @@ -293,32 +277,30 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { return nil } -func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { - results := make([]asset, 0, 64) - for _, name := range sec.Names() { - if isdir, err := sec.IsDir(name); !isdir && err == nil { - if sec.Path == "public" && - strings.HasPrefix(name, "vendor/") && - !c.Bool("include-vendored") { - continue - } - matchName := sec.Path + "/" + name - for _, g := range globs { - if g.Match(matchName) { - results = append(results, asset{ - Section: sec, - Name: name, - Path: sec.Path + "/" + name, - }) - break - } +func collectAssetFilesByPattern(c *cli.Context, globs []glob.Glob, path string, layer *assetfs.Layer) { + fs := assetfs.Layered(layer) + files, err := fs.ListAllFiles(".", true) + if err != nil { + log.Error("Error listing files in %q: %v", path, err) + return + } + for _, name := range files { + if path == "public" && + strings.HasPrefix(name, "vendor/") && + !c.Bool("include-vendored") { + continue + } + matchName := path + "/" + name + for _, g := range globs { + if g.Match(matchName) { + matchedAssetFiles = append(matchedAssetFiles, assetFile{fs: fs, name: name, path: path + "/" + name}) + break } } } - return results } -func getPatterns(args []string) ([]glob.Glob, error) { +func compileCollectPatterns(args []string) ([]glob.Glob, error) { if len(args) == 0 { args = []string{"**"} } @@ -326,7 +308,7 @@ func getPatterns(args []string) ([]glob.Glob, error) { for i := range args { if g, err := glob.Compile(args[i], '/'); err != nil { return nil, fmt.Errorf("'%s': Invalid glob pattern: %w", args[i], err) - } else { + } else { //nolint:revive pat[i] = g } } diff --git a/cmd/embedded_stub.go b/cmd/embedded_stub.go deleted file mode 100644 index 874df06f9d7fc..0000000000000 --- a/cmd/embedded_stub.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !bindata - -package cmd - -import ( - "fmt" - "os" - - "github.com/urfave/cli" -) - -// Cmdembedded represents the available extract sub-command. -var ( - Cmdembedded = cli.Command{ - Name: "embedded", - Usage: "Extract embedded resources", - Description: "A command for extracting embedded resources, like templates and images", - Action: extractorNotImplemented, - } -) - -func extractorNotImplemented(c *cli.Context) error { - err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata") - fmt.Fprintf(os.Stderr, "%s\n", err) - return err -} diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go new file mode 100644 index 0000000000000..d032160a6feca --- /dev/null +++ b/modules/assetfs/layered.go @@ -0,0 +1,260 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "context" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "sort" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/util" + + "github.com/fsnotify/fsnotify" +) + +// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem +type Layer struct { + name string + fs http.FileSystem + localPath string +} + +func (l *Layer) Name() string { + return l.name +} + +// Open opens the named file. The caller is responsible for closing the file. +func (l *Layer) Open(name string) (http.File, error) { + return l.fs.Open(name) +} + +// Local returns a new Layer with the given name, it serves files from the given local path. +func Local(name, base string, sub ...string) *Layer { + // TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before + // Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable. + base, err := filepath.Abs(base) + if err != nil { + // This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths. + panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err)) + } + root := util.FilePathJoinAbs(base, sub...) + return &Layer{name: name, fs: http.Dir(root), localPath: root} +} + +// Bindata returns a new Layer with the given name, it serves files from the given bindata asset. +func Bindata(name string, fs http.FileSystem) *Layer { + return &Layer{name: name, fs: fs} +} + +// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers. +// The first layer is the top layer, and it will be used first. +// If the file is not found in the top layer, it will be searched in the next layer. +type LayeredFS struct { + layers []*Layer +} + +// Layered returns a new LayeredFS with the given layers. The first layer is the top layer. +func Layered(layers ...*Layer) *LayeredFS { + return &LayeredFS{layers: layers} +} + +// Open opens the named file. The caller is responsible for closing the file. +func (l *LayeredFS) Open(name string) (http.File, error) { + for _, layer := range l.layers { + f, err := layer.Open(name) + if err == nil || !os.IsNotExist(err) { + return f, err + } + } + return nil, fs.ErrNotExist +} + +// ReadFile reads the named file. +func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) { + bs, _, err := l.ReadLayeredFile(elems...) + return bs, err +} + +// ReadLayeredFile reads the named file, and returns the layer name. +func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) { + name := util.PathJoinRel(elems...) + for _, layer := range l.layers { + f, err := layer.Open(name) + if os.IsNotExist(err) { + continue + } else if err != nil { + return nil, layer.name, err + } + bs, err := io.ReadAll(f) + _ = f.Close() + return bs, layer.name, err + } + return nil, "", fs.ErrNotExist +} + +func shouldInclude(info fs.FileInfo, fileMode ...bool) bool { + if util.CommonSkip(info.Name()) { + return false + } + if len(fileMode) == 0 { + return true + } else if len(fileMode) == 1 { + return fileMode[0] == !info.Mode().IsDir() + } + panic("too many arguments for fileMode in shouldInclude") +} + +func readDir(layer *Layer, name string) ([]fs.FileInfo, error) { + f, err := layer.Open(name) + if os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + defer f.Close() + return f.Readdir(-1) +} + +// ListFiles lists files/directories in the given directory. The fileMode controls the returned files. +// * omitted: all files and directories will be returned. +// * true: only files will be returned. +// * false: only directories will be returned. +// The returned files are sorted by name. +func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { + fileMap := map[string]bool{} + for _, layer := range l.layers { + infos, err := readDir(layer, name) + if err != nil { + return nil, err + } + for _, info := range infos { + if shouldInclude(info, fileMode...) { + fileMap[info.Name()] = true + } + } + } + files := make([]string, 0, len(fileMap)) + for file := range fileMap { + files = append(files, file) + } + sort.Strings(files) + return files, nil +} + +// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively. +// The fileMode controls the returned files: +// * omitted: all files and directories will be returned. +// * true: only files will be returned. +// * false: only directories will be returned. +// The returned files are sorted by name. +func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) { + return listAllFiles(l.layers, name, fileMode...) +} + +func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) { + fileMap := map[string]bool{} + var list func(dir string) error + list = func(dir string) error { + for _, layer := range layers { + infos, err := readDir(layer, dir) + if err != nil { + return err + } + for _, info := range infos { + path := util.PathJoinRelX(dir, info.Name()) + if shouldInclude(info, fileMode...) { + fileMap[path] = true + } + if info.IsDir() { + if err = list(path); err != nil { + return err + } + } + } + } + return nil + } + if err := list(name); err != nil { + return nil, err + } + var files []string + for file := range fileMap { + files = append(files, file) + } + sort.Strings(files) + return files, nil +} + +// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes. +func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true) + defer finished() + + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Error("Unable to create watcher for asset local file-system: %v", err) + return + } + defer watcher.Close() + + for _, layer := range l.layers { + if layer.localPath == "" { + continue + } + layerDirs, err := listAllFiles([]*Layer{layer}, ".", false) + if err != nil { + log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err) + continue + } + for _, dir := range layerDirs { + if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil { + log.Error("Unable to watch directory %s: %v", dir, err) + } + } + } + + debounce := util.Debounce(100 * time.Millisecond) + + for { + select { + case <-ctx.Done(): + return + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Trace("Watched asset local file-system had event: %v", event) + debounce(callback) + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Error("Watched asset local file-system had error: %v", err) + } + } +} + +// GetFileLayerName returns the name of the first-seen layer that contains the given file. +func (l *LayeredFS) GetFileLayerName(elems ...string) string { + name := util.PathJoinRel(elems...) + for _, layer := range l.layers { + f, err := layer.Open(name) + if os.IsNotExist(err) { + continue + } else if err != nil { + return "" + } + _ = f.Close() + return layer.name + } + return "" +} diff --git a/modules/assetfs/layered_test.go b/modules/assetfs/layered_test.go new file mode 100644 index 0000000000000..b82111e745e81 --- /dev/null +++ b/modules/assetfs/layered_test.go @@ -0,0 +1,109 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLayered(t *testing.T) { + dir := filepath.Join(t.TempDir(), "assetfs-layers") + dir1 := filepath.Join(dir, "l1") + dir2 := filepath.Join(dir, "l2") + + mkdir := func(elems ...string) { + assert.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755)) + } + write := func(content string, elems ...string) { + assert.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644)) + } + + // d1 & f1: only in "l1"; d2 & f2: only in "l2" + // da & fa: in both "l1" and "l2" + mkdir(dir1, "d1") + mkdir(dir1, "da") + mkdir(dir1, "da/sub1") + + mkdir(dir2, "d2") + mkdir(dir2, "da") + mkdir(dir2, "da/sub2") + + write("dummy", dir1, ".DS_Store") + write("f1", dir1, "f1") + write("fa-1", dir1, "fa") + write("d1-f", dir1, "d1/f") + write("da-f-1", dir1, "da/f") + + write("f2", dir2, "f2") + write("fa-2", dir2, "fa") + write("d2-f", dir2, "d2/f") + write("da-f-2", dir2, "da/f") + + assets := Layered(Local("l1", dir1), Local("l2", dir2)) + + f, err := assets.Open("f1") + assert.NoError(t, err) + bs, err := io.ReadAll(f) + assert.NoError(t, err) + assert.EqualValues(t, "f1", string(bs)) + _ = f.Close() + + assertRead := func(expected string, expectedErr error, elems ...string) { + bs, err := assets.ReadFile(elems...) + if err != nil { + assert.ErrorAs(t, err, &expectedErr) + } else { + assert.NoError(t, err) + assert.Equal(t, expected, string(bs)) + } + } + assertRead("f1", nil, "f1") + assertRead("f2", nil, "f2") + assertRead("fa-1", nil, "fa") + + assertRead("d1-f", nil, "d1/f") + assertRead("d2-f", nil, "d2/f") + assertRead("da-f-1", nil, "da/f") + + assertRead("", fs.ErrNotExist, "no-such") + + files, err := assets.ListFiles(".", true) + assert.NoError(t, err) + assert.EqualValues(t, []string{"f1", "f2", "fa"}, files) + + files, err = assets.ListFiles(".", false) + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1", "d2", "da"}, files) + + files, err = assets.ListFiles(".") + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) + + files, err = assets.ListAllFiles(".", true) + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) + + files, err = assets.ListAllFiles(".", false) + assert.NoError(t, err) + assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) + + files, err = assets.ListAllFiles(".") + assert.NoError(t, err) + assert.EqualValues(t, []string{ + "d1", "d1/f", + "d2", "d2/f", + "da", "da/f", "da/sub1", "da/sub2", + "f1", "f2", "fa", + }, files) + + assert.Empty(t, assets.GetFileLayerName("no-such")) + assert.EqualValues(t, "l1", assets.GetFileLayerName("f1")) + assert.EqualValues(t, "l2", assets.GetFileLayerName("f2")) +} diff --git a/modules/auth/pam/pam_stub.go b/modules/auth/pam/pam_stub.go index a48e89860edb0..3631eeeda7b5a 100644 --- a/modules/auth/pam/pam_stub.go +++ b/modules/auth/pam/pam_stub.go @@ -14,5 +14,9 @@ var Supported = false // Auth not supported lack of pam tag func Auth(serviceName, userName, passwd string) (string, error) { - return "", errors.New("PAM not supported") + // bypass the lint on callers: SA4023: this comparison is always true (staticcheck) + if !Supported { + return "", errors.New("PAM not supported") + } + return "", nil } diff --git a/modules/context/context.go b/modules/context/context.go index bd561be0f5326..e2e120ba384ad 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -240,19 +240,15 @@ func (ctx *Context) HTML(status int, name base.TplName) { } line, _ := strconv.Atoi(lineStr) // Cannot error out as groups[2] is [1-9][0-9]* pos, _ := strconv.Atoi(posStr) // Cannot error out as groups[3] is [1-9][0-9]* - filename, filenameErr := templates.GetAssetFilename("templates/" + errorTemplateName + ".tmpl") - if filenameErr != nil { - filename = "(template) " + errorTemplateName - } + assetLayerName := templates.AssetFS().GetFileLayerName(errorTemplateName + ".tmpl") + filename := fmt.Sprintf("(%s) %s", assetLayerName, errorTemplateName) if errorTemplateName != string(name) { filename += " (subtemplate of " + string(name) + ")" } err = fmt.Errorf("failed to render %s, error: %w:\n%s", filename, err, templates.GetLineFromTemplate(errorTemplateName, line, target, pos)) } else { - filename, filenameErr := templates.GetAssetFilename("templates/" + execErr.Name + ".tmpl") - if filenameErr != nil { - filename = "(template) " + execErr.Name - } + assetLayerName := templates.AssetFS().GetFileLayerName(execErr.Name + ".tmpl") + filename := fmt.Sprintf("(%s) %s", assetLayerName, execErr.Name) if execErr.Name != string(name) { filename += " (subtemplate of " + string(name) + ")" } diff --git a/modules/doctor/paths.go b/modules/doctor/paths.go index 7b1c6ce9ad1c8..d7bf2539e760a 100644 --- a/modules/doctor/paths.go +++ b/modules/doctor/paths.go @@ -9,7 +9,6 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" ) @@ -79,7 +78,7 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo {"Log Root Path", setting.Log.RootPath, true, true, true}, } - if options.IsDynamic() { + if !setting.HasBuiltinBindata { configurationFiles = append(configurationFiles, configurationFile{"Static File Root Path", setting.StaticRootPath, true, true, false}) } diff --git a/modules/options/base.go b/modules/options/base.go index 7882ed008159a..6c6e3839f462d 100644 --- a/modules/options/base.go +++ b/modules/options/base.go @@ -4,131 +4,39 @@ package options import ( - "fmt" - "io/fs" - "os" - "path/filepath" - - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -var directories = make(directorySet) +func CustomAssets() *assetfs.Layer { + return assetfs.Local("custom", setting.CustomPath, "options") +} + +func AssetFS() *assetfs.LayeredFS { + return assetfs.Layered(CustomAssets(), BuiltinAssets()) +} // Locale reads the content of a specific locale from static/bindata or custom path. func Locale(name string) ([]byte, error) { - return fileFromOptionsDir("locale", name) + return AssetFS().ReadFile("locale", name) } // Readme reads the content of a specific readme from static/bindata or custom path. func Readme(name string) ([]byte, error) { - return fileFromOptionsDir("readme", name) + return AssetFS().ReadFile("readme", name) } // Gitignore reads the content of a gitignore locale from static/bindata or custom path. func Gitignore(name string) ([]byte, error) { - return fileFromOptionsDir("gitignore", name) + return AssetFS().ReadFile("gitignore", name) } // License reads the content of a specific license from static/bindata or custom path. func License(name string) ([]byte, error) { - return fileFromOptionsDir("license", name) + return AssetFS().ReadFile("license", name) } // Labels reads the content of a specific labels from static/bindata or custom path. func Labels(name string) ([]byte, error) { - return fileFromOptionsDir("label", name) -} - -// WalkLocales reads the content of a specific locale -func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { - if IsDynamic() { - if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to walk locales. Error: %w", err) - } - } - - if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to walk locales. Error: %w", err) - } - return nil -} - -func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - // name is the path relative to the root - name := path[len(root):] - if len(name) > 0 && name[0] == '/' { - name = name[1:] - } - if err != nil { - if os.IsNotExist(err) { - return callback(path, name, d, err) - } - return err - } - if util.CommonSkip(d.Name()) { - if d.IsDir() { - return fs.SkipDir - } - return nil - } - return callback(path, name, d, err) - }); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("unable to get files for assets in %s: %w", root, err) - } - return nil -} - -// mustLocalPathAbs coverts a path to absolute path -// FIXME: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before -func mustLocalPathAbs(s string) string { - abs, err := filepath.Abs(s) - if err != nil { - // This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths. - log.Fatal("Unable to get absolute path for %q: %v", s, err) - } - return abs -} - -func joinLocalPaths(baseDirs []string, subDir string, elems ...string) (paths []string) { - abs := make([]string, len(elems)+2) - abs[1] = subDir - copy(abs[2:], elems) - for _, baseDir := range baseDirs { - abs[0] = mustLocalPathAbs(baseDir) - paths = append(paths, util.FilePathJoinAbs(abs...)) - } - return paths -} - -func listLocalDirIfExist(baseDirs []string, subDir string, elems ...string) (files []string, err error) { - for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) { - isDir, err := util.IsDir(localPath) - if err != nil { - return nil, fmt.Errorf("unable to check if path %q is a directory. %w", localPath, err) - } else if !isDir { - continue - } - - dirFiles, err := util.StatDir(localPath, true) - if err != nil { - return nil, fmt.Errorf("unable to read directory %q. %w", localPath, err) - } - files = append(files, dirFiles...) - } - return files, nil -} - -func readLocalFile(baseDirs []string, subDir string, elems ...string) ([]byte, error) { - for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) { - data, err := os.ReadFile(localPath) - if err == nil { - return data, nil - } else if !os.IsNotExist(err) { - log.Error("Unable to read file %q. Error: %v", localPath, err) - } - } - return nil, os.ErrNotExist + return AssetFS().ReadFile("label", name) } diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go index 3d6261983f2de..085492d11c876 100644 --- a/modules/options/dynamic.go +++ b/modules/options/dynamic.go @@ -6,29 +6,10 @@ package options import ( + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" ) -// Dir returns all files from static or custom directory. -func Dir(name string) ([]string, error) { - if directories.Filled(name) { - return directories.Get(name), nil - } - - result, err := listLocalDirIfExist([]string{setting.CustomPath, setting.StaticRootPath}, "options", name) - if err != nil { - return nil, err - } - - return directories.AddAndGet(name, result), nil -} - -// fileFromOptionsDir is a helper to read files from custom or static path. -func fileFromOptionsDir(elems ...string) ([]byte, error) { - return readLocalFile([]string{setting.CustomPath, setting.StaticRootPath}, "options", elems...) -} - -// IsDynamic will return false when using embedded data (-tags bindata) -func IsDynamic() bool { - return true +func BuiltinAssets() *assetfs.Layer { + return assetfs.Local("builtin(static)", setting.StaticRootPath, "options") } diff --git a/modules/options/options.go b/modules/options/options.go deleted file mode 100644 index 17a8fa482e42e..0000000000000 --- a/modules/options/options.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package options - -type directorySet map[string][]string - -func (s directorySet) Add(key string, value []string) { - _, ok := s[key] - - if !ok { - s[key] = make([]string, 0, len(value)) - } - - s[key] = append(s[key], value...) -} - -func (s directorySet) Get(key string) []string { - _, ok := s[key] - - if ok { - result := []string{} - seen := map[string]string{} - - for _, val := range s[key] { - if _, ok := seen[val]; !ok { - result = append(result, val) - seen[val] = val - } - } - - return result - } - - return []string{} -} - -func (s directorySet) AddAndGet(key string, value []string) []string { - s.Add(key, value) - return s.Get(key) -} - -func (s directorySet) Filled(key string) bool { - return len(s[key]) > 0 -} diff --git a/modules/options/static.go b/modules/options/static.go index 0482dea6817ce..72b28e990e777 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -6,98 +6,9 @@ package options import ( - "fmt" - "io" - - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/assetfs" ) -// Dir returns all files from custom directory or bindata. -func Dir(name string) ([]string, error) { - if directories.Filled(name) { - return directories.Get(name), nil - } - - result, err := listLocalDirIfExist([]string{setting.CustomPath}, "options", name) - if err != nil { - return nil, err - } - - files, err := AssetDir(name) - if err != nil { - return []string{}, fmt.Errorf("unable to read embedded directory %q. %w", name, err) - } - - result = append(result, files...) - return directories.AddAndGet(name, result), nil -} - -func AssetDir(dirName string) ([]string, error) { - d, err := Assets.Open(dirName) - if err != nil { - return nil, err - } - defer d.Close() - - files, err := d.Readdir(-1) - if err != nil { - return nil, err - } - results := make([]string, 0, len(files)) - for _, file := range files { - results = append(results, file.Name()) - } - return results, nil -} - -// fileFromOptionsDir is a helper to read files from custom path or bindata. -func fileFromOptionsDir(elems ...string) ([]byte, error) { - // only try custom dir, no static dir - if data, err := readLocalFile([]string{setting.CustomPath}, "options", elems...); err == nil { - return data, nil - } - - f, err := Assets.Open(util.PathJoinRelX(elems...)) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -func Asset(name string) ([]byte, error) { - f, err := Assets.Open("/" + name) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -func AssetNames() []string { - realFS := Assets.(vfsgen۰FS) - results := make([]string, 0, len(realFS)) - for k := range realFS { - results = append(results, k[1:]) - } - return results -} - -func AssetIsDir(name string) (bool, error) { - if f, err := Assets.Open("/" + name); err != nil { - return false, err - } else { - defer f.Close() - if fi, err := f.Stat(); err != nil { - return false, err - } else { - return fi.IsDir(), nil - } - } -} - -// IsDynamic will return false when using embedded data (-tags bindata) -func IsDynamic() bool { - return false +func BuiltinAssets() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/public/public.go b/modules/public/public.go index 2c96cf9e763b9..0c0e6dc1cc804 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -4,11 +4,15 @@ package public import ( + "bytes" + "io" "net/http" "os" - "path/filepath" + "path" "strings" + "time" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" @@ -16,55 +20,31 @@ import ( "code.gitea.io/gitea/modules/util" ) -// Options represents the available options to configure the handler. -type Options struct { - Directory string - Prefix string - CorsHandler func(http.Handler) http.Handler +func CustomAssets() *assetfs.Layer { + return assetfs.Local("custom", setting.CustomPath, "public") } -// AssetsURLPathPrefix is the path prefix for static asset files -const AssetsURLPathPrefix = "/assets/" +func AssetFS() *assetfs.LayeredFS { + return assetfs.Layered(CustomAssets(), BuiltinAssets()) +} // AssetsHandlerFunc implements the static handler for serving custom or original assets. -func AssetsHandlerFunc(opts *Options) http.HandlerFunc { - custPath := filepath.Join(setting.CustomPath, "public") - if !filepath.IsAbs(custPath) { - custPath = filepath.Join(setting.AppWorkPath, custPath) - } - if !filepath.IsAbs(opts.Directory) { - opts.Directory = filepath.Join(setting.AppWorkPath, opts.Directory) - } - if !strings.HasSuffix(opts.Prefix, "/") { - opts.Prefix += "/" - } - +func AssetsHandlerFunc(prefix string) http.HandlerFunc { + assetFS := AssetFS() + prefix = strings.TrimSuffix(prefix, "/") + "/" return func(resp http.ResponseWriter, req *http.Request) { - if req.Method != "GET" && req.Method != "HEAD" { - resp.WriteHeader(http.StatusNotFound) + subPath := req.URL.Path + if !strings.HasPrefix(subPath, prefix) { return } + subPath = strings.TrimPrefix(subPath, prefix) - if opts.CorsHandler != nil { - var corsSent bool - opts.CorsHandler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - corsSent = true - })).ServeHTTP(resp, req) - // If CORS is not sent, the response must have been written by other handlers - if !corsSent { - return - } - } - - file := req.URL.Path[len(opts.Prefix):] - - // custom files - if opts.handle(resp, req, http.Dir(custPath), file) { + if req.Method != "GET" && req.Method != "HEAD" { + resp.WriteHeader(http.StatusNotFound) return } - // internal files - if opts.handle(resp, req, fileSystem(opts.Directory), file) { + if handleRequest(resp, req, assetFS, subPath) { return } @@ -85,13 +65,13 @@ func parseAcceptEncoding(val string) container.Set[string] { // setWellKnownContentType will set the Content-Type if the file is a well-known type. // See the comments of detectWellKnownMimeType func setWellKnownContentType(w http.ResponseWriter, file string) { - mimeType := detectWellKnownMimeType(filepath.Ext(file)) + mimeType := detectWellKnownMimeType(path.Ext(file)) if mimeType != "" { w.Header().Set("Content-Type", mimeType) } } -func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { +func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { // actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here f, err := fs.Open(util.PathJoinRelX(file)) if err != nil { @@ -121,8 +101,34 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.Fi return true } - setWellKnownContentType(w, file) - serveContent(w, req, fi, fi.ModTime(), f) return true } + +type GzipBytesProvider interface { + GzipBytes() []byte +} + +// serveContent serve http content +func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { + setWellKnownContentType(w, fi.Name()) + + encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) + if encodings.Contains("gzip") { + // try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo) + if compressed, ok := fi.(GzipBytesProvider); ok { + rdGzip := bytes.NewReader(compressed.GzipBytes()) + // all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name + // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data + if w.Header().Get("Content-Type") == "" { + w.Header().Set("Content-Type", "application/octet-stream") + } + w.Header().Set("Content-Encoding", "gzip") + http.ServeContent(w, req, fi.Name(), modtime, rdGzip) + return + } + } + + http.ServeContent(w, req, fi.Name(), modtime, content) + return +} diff --git a/modules/public/serve_dynamic.go b/modules/public/serve_dynamic.go index cd74ee574366c..a668b17c34457 100644 --- a/modules/public/serve_dynamic.go +++ b/modules/public/serve_dynamic.go @@ -6,17 +6,10 @@ package public import ( - "io" - "net/http" - "os" - "time" + "code.gitea.io/gitea/modules/assetfs" + "code.gitea.io/gitea/modules/setting" ) -func fileSystem(dir string) http.FileSystem { - return http.Dir(dir) -} - -// serveContent serve http content -func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { - http.ServeContent(w, req, fi.Name(), modtime, content) +func BuiltinAssets() *assetfs.Layer { + return assetfs.Local("builtin(static)", setting.StaticRootPath, "public") } diff --git a/modules/public/serve_static.go b/modules/public/serve_static.go index e85ca79253f74..e79085021eab1 100644 --- a/modules/public/serve_static.go +++ b/modules/public/serve_static.go @@ -6,75 +6,19 @@ package public import ( - "bytes" - "io" - "net/http" - "os" - "path/filepath" "time" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/timeutil" ) +var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil) + // GlobalModTime provide a global mod time for embedded asset files func GlobalModTime(filename string) time.Time { return timeutil.GetExecutableModTime() } -func fileSystem(dir string) http.FileSystem { - return Assets -} - -func Asset(name string) ([]byte, error) { - f, err := Assets.Open("/" + name) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -func AssetNames() []string { - realFS := Assets.(vfsgen۰FS) - results := make([]string, 0, len(realFS)) - for k := range realFS { - results = append(results, k[1:]) - } - return results -} - -func AssetIsDir(name string) (bool, error) { - if f, err := Assets.Open("/" + name); err != nil { - return false, err - } else { - defer f.Close() - if fi, err := f.Stat(); err != nil { - return false, err - } else { - return fi.IsDir(), nil - } - } -} - -// serveContent serve http content -func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { - encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) - if encodings.Contains("gzip") { - if cf, ok := fi.(*vfsgen۰CompressedFileInfo); ok { - rdGzip := bytes.NewReader(cf.GzipBytes()) - // all static files are managed by Gitea, so we can make sure every file has the correct ext name - // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data - mimeType := detectWellKnownMimeType(filepath.Ext(fi.Name())) - if mimeType == "" { - mimeType = "application/octet-stream" - } - w.Header().Set("Content-Type", mimeType) - w.Header().Set("Content-Encoding", "gzip") - http.ServeContent(w, req, fi.Name(), modtime, rdGzip) - return - } - } - - http.ServeContent(w, req, fi.Name(), modtime, content) - return +func BuiltinAssets() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/repository/init.go b/modules/repository/init.go index 38dd8a0c4f969..cb353f24968a7 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -79,7 +79,7 @@ func LoadRepoConfig() error { typeFiles := make([]optionFileList, len(types)) for i, t := range types { var err error - if typeFiles[i].all, err = options.Dir(t); err != nil { + if typeFiles[i].all, err = options.AssetFS().ListFiles(t, true); err != nil { return fmt.Errorf("failed to list %s files: %w", t, err) } sort.Strings(typeFiles[i].all) diff --git a/modules/setting/asset_dynamic.go b/modules/setting/asset_dynamic.go new file mode 100644 index 0000000000000..2eb28833736b4 --- /dev/null +++ b/modules/setting/asset_dynamic.go @@ -0,0 +1,8 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build !bindata + +package setting + +const HasBuiltinBindata = false diff --git a/modules/setting/asset_static.go b/modules/setting/asset_static.go new file mode 100644 index 0000000000000..889fca9342415 --- /dev/null +++ b/modules/setting/asset_static.go @@ -0,0 +1,8 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build bindata + +package setting + +const HasBuiltinBindata = true diff --git a/modules/svg/discover_bindata.go b/modules/svg/discover_bindata.go deleted file mode 100644 index b6abd294f17d4..0000000000000 --- a/modules/svg/discover_bindata.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package svg - -import ( - "path/filepath" - - "code.gitea.io/gitea/modules/public" -) - -// Discover returns a map of discovered SVG icons in bindata -func Discover() map[string]string { - svgs := make(map[string]string) - - for _, file := range public.AssetNames() { - matched, _ := filepath.Match("img/svg/*.svg", file) - if matched { - content, err := public.Asset(file) - if err == nil { - filename := filepath.Base(file) - svgs[filename[:len(filename)-4]] = string(content) - } - } - } - - return svgs -} diff --git a/modules/svg/discover_nobindata.go b/modules/svg/discover_nobindata.go deleted file mode 100644 index da7ab7b98f5de..0000000000000 --- a/modules/svg/discover_nobindata.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !bindata - -package svg - -import ( - "os" - "path/filepath" - - "code.gitea.io/gitea/modules/setting" -) - -// Discover returns a map of discovered SVG icons in the file system -func Discover() map[string]string { - svgs := make(map[string]string) - - files, _ := filepath.Glob(filepath.Join(setting.StaticRootPath, "public", "img", "svg", "*.svg")) - for _, file := range files { - content, err := os.ReadFile(file) - if err == nil { - filename := filepath.Base(file) - svgs[filename[:len(filename)-4]] = string(content) - } - } - - return svgs -} diff --git a/modules/svg/svg.go b/modules/svg/svg.go index b74ee3535800d..071340764e049 100644 --- a/modules/svg/svg.go +++ b/modules/svg/svg.go @@ -6,15 +6,18 @@ package svg import ( "fmt" "html/template" + "path" "regexp" "strings" "code.gitea.io/gitea/modules/html" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/public" ) var ( // SVGs contains discovered SVGs - SVGs map[string]string + SVGs = map[string]string{} widthRe = regexp.MustCompile(`width="[0-9]+?"`) heightRe = regexp.MustCompile(`height="[0-9]+?"`) @@ -23,17 +26,29 @@ var ( const defaultSize = 16 // Init discovers SVGs and populates the `SVGs` variable -func Init() { - SVGs = Discover() +func Init() error { + files, err := public.AssetFS().ListFiles("img/svg") + if err != nil { + return err + } // Remove `xmlns` because inline SVG does not need it - r := regexp.MustCompile(`(]*?)\s+xmlns="[^"]*"`) - for name, svg := range SVGs { - SVGs[name] = r.ReplaceAllString(svg, "$1") + reXmlns := regexp.MustCompile(`(]*?)\s+xmlns="[^"]*"`) + for _, file := range files { + if path.Ext(file) != ".svg" { + continue + } + bs, err := public.AssetFS().ReadFile("img/svg", file) + if err != nil { + log.Error("Failed to read SVG file %s: %v", file, err) + } else { + SVGs[file[:len(file)-4]] = reXmlns.ReplaceAllString(string(bs), "$1") + } } + return nil } -// Render render icons - arguments icon name (string), size (int), class (string) +// RenderHTML renders icons - arguments icon name (string), size (int), class (string) func RenderHTML(icon string, others ...interface{}) template.HTML { size, class := html.ParseSizeAndClass(defaultSize, "", others...) diff --git a/modules/templates/base.go b/modules/templates/base.go index e0f8350afb11b..e95ce31cfcab0 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -4,14 +4,10 @@ package templates import ( - "fmt" - "io/fs" - "os" - "path/filepath" "strings" "time" - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -47,81 +43,30 @@ func BaseVars() Vars { } } -func getDirTemplateAssetNames(dir string) []string { - return getDirAssetNames(dir, false) +func AssetFS() *assetfs.LayeredFS { + return assetfs.Layered(CustomAssets(), BuiltinAssets()) } -func getDirAssetNames(dir string, mailer bool) []string { - var tmpls []string - - if mailer { - dir += filepath.Join(dir, "mail") - } - f, err := os.Stat(dir) - if err != nil { - if os.IsNotExist(err) { - return tmpls - } - log.Warn("Unable to check if templates dir %s is a directory. Error: %v", dir, err) - return tmpls - } - if !f.IsDir() { - log.Warn("Templates dir %s is a not directory.", dir) - return tmpls - } +func CustomAssets() *assetfs.Layer { + return assetfs.Local("custom", setting.CustomPath, "templates") +} - files, err := util.StatDir(dir) +func ListWebTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) { + files, err := assets.ListAllFiles(".", true) if err != nil { - log.Warn("Failed to read %s templates dir. %v", dir, err) - return tmpls + return nil, err } - - prefix := "templates/" - if mailer { - prefix += "mail/" - } - for _, filePath := range files { - if !mailer && strings.HasPrefix(filePath, "mail/") { - continue - } - - if !strings.HasSuffix(filePath, ".tmpl") { - continue - } - - tmpls = append(tmpls, prefix+filePath) - } - return tmpls + return util.SliceRemoveAllFunc(files, func(file string) bool { + return strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl") + }), nil } -func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error { - mailRoot := filepath.Join(root, "mail") - if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - name := path[len(root):] - if len(name) > 0 && name[0] == '/' { - name = name[1:] - } - if err != nil { - if os.IsNotExist(err) { - return callback(path, name, d, err) - } - return err - } - if skipMail && path == mailRoot && d.IsDir() { - return fs.SkipDir - } - if util.CommonSkip(d.Name()) { - if d.IsDir() { - return fs.SkipDir - } - return nil - } - if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() { - return callback(path, name, d, err) - } - return nil - }); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("unable to get files for template assets in %s: %w", root, err) +func ListMailTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) { + files, err := assets.ListAllFiles(".", true) + if err != nil { + return nil, err } - return nil + return util.SliceRemoveAllFunc(files, func(file string) bool { + return !strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl") + }), nil } diff --git a/modules/templates/dynamic.go b/modules/templates/dynamic.go index 2f4f542e724e6..e1babd83c9438 100644 --- a/modules/templates/dynamic.go +++ b/modules/templates/dynamic.go @@ -6,76 +6,10 @@ package templates import ( - "io/fs" - "os" - "path/filepath" - + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" ) -// GetAsset returns asset content via name -func GetAsset(name string) ([]byte, error) { - bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name)) - if err != nil && !os.IsNotExist(err) { - return nil, err - } else if err == nil { - return bs, nil - } - - return os.ReadFile(filepath.Join(setting.StaticRootPath, name)) -} - -// GetAssetFilename returns the filename of the provided asset -func GetAssetFilename(name string) (string, error) { - filename := filepath.Join(setting.CustomPath, name) - _, err := os.Stat(filename) - if err != nil && !os.IsNotExist(err) { - return filename, err - } else if err == nil { - return filename, nil - } - - filename = filepath.Join(setting.StaticRootPath, name) - _, err = os.Stat(filename) - return filename, err -} - -// walkTemplateFiles calls a callback for each template asset -func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { - return err - } - if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// GetTemplateAssetNames returns list of template names -func GetTemplateAssetNames() []string { - tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates")) - tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates")) - return append(tmpls, tmpls2...) -} - -func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { - return err - } - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// BuiltinAsset will read the provided asset from the embedded assets -// (This always returns os.ErrNotExist) -func BuiltinAsset(name string) ([]byte, error) { - return nil, os.ErrNotExist -} - -// BuiltinAssetNames returns the names of the embedded assets -// (This always returns nil) -func BuiltinAssetNames() []string { - return nil +func BuiltinAssets() *assetfs.Layer { + return assetfs.Local("builtin(static)", setting.StaticRootPath, "templates") } diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 4e7b09a9eccec..26dd365e4ca36 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -21,7 +21,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/watcher" ) var ( @@ -66,20 +65,23 @@ func (h *HTMLRender) TemplateLookup(name string) (*template.Template, error) { } func (h *HTMLRender) CompileTemplates() error { - dirPrefix := "templates/" extSuffix := ".tmpl" tmpls := template.New("") - for _, path := range GetTemplateAssetNames() { - if !strings.HasSuffix(path, extSuffix) { + assets := AssetFS() + files, err := ListWebTemplateAssetNames(assets) + if err != nil { + return nil + } + for _, file := range files { + if !strings.HasSuffix(file, extSuffix) { continue } - name := strings.TrimPrefix(path, dirPrefix) - name = strings.TrimSuffix(name, extSuffix) + name := strings.TrimSuffix(file, extSuffix) tmpl := tmpls.New(filepath.ToSlash(name)) for _, fm := range NewFuncMap() { tmpl.Funcs(fm) } - buf, err := GetAsset(path) + buf, err := assets.ReadFile(file) if err != nil { return err } @@ -112,13 +114,10 @@ func HTMLRenderer(ctx context.Context) (context.Context, *HTMLRender) { log.Fatal("HTMLRenderer error: %v", err) } if !setting.IsProd { - watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{ - PathsCallback: walkTemplateFiles, - BetweenCallback: func() { - if err := renderer.CompileTemplates(); err != nil { - log.Error("Template error: %v\n%s", err, log.Stack(2)) - } - }, + go AssetFS().WatchLocalChanges(ctx, func() { + if err := renderer.CompileTemplates(); err != nil { + log.Error("Template error: %v\n%s", err, log.Stack(2)) + } }) } return context.WithValue(ctx, rendererKey, renderer), renderer @@ -138,14 +137,8 @@ func handleGenericTemplateError(err error) (string, []interface{}) { } templateName, lineNumberStr, message := groups[1], groups[2], groups[3] - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, "", -1) return "PANIC: Unable to compile templates!\n%s in template file %s at line %d:\n\n%s\nStacktrace:\n\n%s", []interface{}{message, filename, lineNumber, log.NewColoredValue(line, log.Reset), log.Stack(2)} @@ -158,16 +151,9 @@ func handleNotDefinedPanicError(err error) (string, []interface{}) { } templateName, lineNumberStr, functionName := groups[1], groups[2], groups[3] - functionName, _ = strconv.Unquote(`"` + functionName + `"`) - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, functionName, -1) return "PANIC: Unable to compile templates!\nUndefined function %q in template file %s at line %d:\n\n%s", []interface{}{functionName, filename, lineNumber, log.NewColoredValue(line, log.Reset)} @@ -181,14 +167,8 @@ func handleUnexpected(err error) (string, []interface{}) { templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3] unexpected, _ = strconv.Unquote(`"` + unexpected + `"`) - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1) return "PANIC: Unable to compile templates!\nUnexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)} @@ -201,14 +181,8 @@ func handleExpectedEnd(err error) (string, []interface{}) { } templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3] - - filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") - if assetErr != nil { - return "", nil - } - + filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) lineNumber, _ := strconv.Atoi(lineNumberStr) - line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1) return "PANIC: Unable to compile templates!\nMissing end with unexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)} @@ -218,7 +192,7 @@ const dashSeparator = "--------------------------------------------------------- // GetLineFromTemplate returns a line from a template with some context func GetLineFromTemplate(templateName string, targetLineNum int, target string, position int) string { - bs, err := GetAsset("templates/" + templateName + ".tmpl") + bs, err := AssetFS().ReadFile(templateName + ".tmpl") if err != nil { return fmt.Sprintf("(unable to read template file: %v)", err) } diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go index d0c49e1025bf0..280ac0e587e26 100644 --- a/modules/templates/mailer.go +++ b/modules/templates/mailer.go @@ -6,15 +6,12 @@ package templates import ( "context" "html/template" - "io/fs" - "os" "strings" texttmpl "text/template" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/watcher" ) // mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject @@ -62,54 +59,23 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { bodyTemplates.Funcs(funcs) } + assetFS := AssetFS() refreshTemplates := func() { - for _, assetPath := range BuiltinAssetNames() { - if !strings.HasPrefix(assetPath, "mail/") { - continue - } - - if !strings.HasSuffix(assetPath, ".tmpl") { - continue - } - - content, err := BuiltinAsset(assetPath) - if err != nil { - log.Warn("Failed to read embedded %s template. %v", assetPath, err) - continue - } - - assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") - - log.Trace("Adding built-in mailer template for %s", assetName) - buildSubjectBodyTemplate(subjectTemplates, - bodyTemplates, - assetName, - content) + assetPaths, err := ListMailTemplateAssetNames(assetFS) + if err != nil { + log.Error("Failed to list mail templates: %v", err) + return } - if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - - content, err := os.ReadFile(path) + for _, assetPath := range assetPaths { + content, layerName, err := assetFS.ReadLayeredFile(assetPath) if err != nil { - log.Warn("Failed to read custom %s template. %v", path, err) - return nil + log.Warn("Failed to read mail template %s by %s: %v", assetPath, layerName, err) + continue } - - assetName := strings.TrimSuffix(name, ".tmpl") - log.Trace("Adding mailer template for %s from %q", assetName, path) - buildSubjectBodyTemplate(subjectTemplates, - bodyTemplates, - assetName, - content) - return nil - }); err != nil && !os.IsNotExist(err) { - log.Warn("Error whilst walking mailer templates directories. %v", err) + tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") + log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName) + buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content) } } @@ -118,10 +84,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { if !setting.IsProd { // Now subjectTemplates and bodyTemplates are both synchronized // thus it is safe to call refresh from a different goroutine - watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{ - PathsCallback: walkMailerTemplates, - BetweenCallback: refreshTemplates, - }) + go assetFS.WatchLocalChanges(ctx, refreshTemplates) } return subjectTemplates, bodyTemplates diff --git a/modules/templates/static.go b/modules/templates/static.go index 7ebb327ae6ebd..b5a7e561ec065 100644 --- a/modules/templates/static.go +++ b/modules/templates/static.go @@ -6,114 +6,17 @@ package templates import ( - "html/template" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - texttmpl "text/template" "time" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/timeutil" ) -var ( - subjectTemplates = texttmpl.New("") - bodyTemplates = template.New("") -) - // GlobalModTime provide a global mod time for embedded asset files func GlobalModTime(filename string) time.Time { return timeutil.GetExecutableModTime() } -// GetAssetFilename returns the filename of the provided asset -func GetAssetFilename(name string) (string, error) { - filename := filepath.Join(setting.CustomPath, name) - _, err := os.Stat(filename) - if err != nil && !os.IsNotExist(err) { - return name, err - } else if err == nil { - return filename, nil - } - return "(builtin) " + name, nil -} - -// GetAsset get a special asset, only for chi -func GetAsset(name string) ([]byte, error) { - bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name)) - if err != nil && !os.IsNotExist(err) { - return nil, err - } else if err == nil { - return bs, nil - } - return BuiltinAsset(strings.TrimPrefix(name, "templates/")) -} - -// GetFiles calls a callback for each template asset -func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// GetTemplateAssetNames only for chi -func GetTemplateAssetNames() []string { - realFS := Assets.(vfsgen۰FS) - tmpls := make([]string, 0, len(realFS)) - for k := range realFS { - if strings.HasPrefix(k, "/mail/") { - continue - } - tmpls = append(tmpls, "templates/"+k[1:]) - } - - customDir := path.Join(setting.CustomPath, "templates") - customTmpls := getDirTemplateAssetNames(customDir) - return append(tmpls, customTmpls...) -} - -func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// BuiltinAsset reads the provided asset from the builtin embedded assets -func BuiltinAsset(name string) ([]byte, error) { - f, err := Assets.Open("/" + name) - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - -// BuiltinAssetNames returns the names of the built-in embedded assets -func BuiltinAssetNames() []string { - realFS := Assets.(vfsgen۰FS) - results := make([]string, 0, len(realFS)) - for k := range realFS { - results = append(results, k[1:]) - } - return results -} - -// BuiltinAssetIsDir returns if a provided asset is a directory -func BuiltinAssetIsDir(name string) (bool, error) { - if f, err := Assets.Open("/" + name); err != nil { - return false, err - } else { - defer f.Close() - if fi, err := f.Stat(); err != nil { - return false, err - } else { - return fi.IsDir(), nil - } - } +func BuiltinAssets() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", Assets) } diff --git a/modules/translation/translation.go b/modules/translation/translation.go index 3165390c3238c..331da0f965c38 100644 --- a/modules/translation/translation.go +++ b/modules/translation/translation.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation/i18n" - "code.gitea.io/gitea/modules/watcher" "golang.org/x/text/language" ) @@ -58,7 +57,7 @@ func InitLocales(ctx context.Context) { refreshLocales := func() { i18n.ResetDefaultLocales() - localeNames, err := options.Dir("locale") + localeNames, err := options.AssetFS().ListFiles("locale", true) if err != nil { log.Fatal("Failed to list locale files: %v", err) } @@ -118,13 +117,10 @@ func InitLocales(ctx context.Context) { }) if !setting.IsProd { - watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{ - PathsCallback: options.WalkLocales, - BetweenCallback: func() { - lock.Lock() - defer lock.Unlock() - refreshLocales() - }, + go options.AssetFS().WatchLocalChanges(ctx, func() { + lock.Lock() + defer lock.Unlock() + refreshLocales() }) } } diff --git a/modules/util/path.go b/modules/util/path.go index 37d06e9813e07..1a68bc748802d 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -74,29 +74,28 @@ const pathSeparator = string(os.PathSeparator) // // {`/foo`, ``, `bar`} => `/foo/bar` // {`/foo`, `..`, `bar`} => `/foo/bar` -func FilePathJoinAbs(elem ...string) string { - elems := make([]string, len(elem)) +func FilePathJoinAbs(base string, sub ...string) string { + elems := make([]string, 1, len(sub)+1) - // POISX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators + // POSIX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators // to keep the behavior consistent, we do not allow `\` in file names, replace all `\` with `/` if isOSWindows() { - elems[0] = filepath.Clean(elem[0]) + elems[0] = filepath.Clean(base) } else { - elems[0] = filepath.Clean(strings.ReplaceAll(elem[0], "\\", pathSeparator)) + elems[0] = filepath.Clean(strings.ReplaceAll(base, "\\", pathSeparator)) } if !filepath.IsAbs(elems[0]) { // This shouldn't happen. If there is really necessary to pass in relative path, return the full path with filepath.Abs() instead panic(fmt.Sprintf("FilePathJoinAbs: %q (for path %v) is not absolute, do not guess a relative path based on current working directory", elems[0], elems)) } - - for i := 1; i < len(elem); i++ { - if elem[i] == "" { + for _, s := range sub { + if s == "" { continue } if isOSWindows() { - elems[i] = filepath.Clean(pathSeparator + elem[i]) + elems = append(elems, filepath.Clean(pathSeparator+s)) } else { - elems[i] = filepath.Clean(pathSeparator + strings.ReplaceAll(elem[i], "\\", pathSeparator)) + elems = append(elems, filepath.Clean(pathSeparator+strings.ReplaceAll(s, "\\", pathSeparator))) } } // the elems[0] must be an absolute path, just join them together diff --git a/modules/util/path_test.go b/modules/util/path_test.go index 1d27c9bf0c0f8..6a38bf4ace863 100644 --- a/modules/util/path_test.go +++ b/modules/util/path_test.go @@ -207,6 +207,6 @@ func TestCleanPath(t *testing.T) { } } for _, c := range cases { - assert.Equal(t, c.expected, FilePathJoinAbs(c.elems...), "case: %v", c.elems) + assert.Equal(t, c.expected, FilePathJoinAbs(c.elems[0], c.elems[1:]...), "case: %v", c.elems) } } diff --git a/modules/util/timer.go b/modules/util/timer.go index daf96bda7e236..d598fde73ad27 100644 --- a/modules/util/timer.go +++ b/modules/util/timer.go @@ -4,6 +4,7 @@ package util import ( + "sync" "time" ) @@ -18,3 +19,30 @@ func StopTimer(t *time.Timer) bool { } return stopped } + +func Debounce(d time.Duration) func(f func()) { + type debouncer struct { + mu sync.Mutex + t *time.Timer + } + db := &debouncer{} + + return func(f func()) { + db.mu.Lock() + defer db.mu.Unlock() + + if db.t != nil { + db.t.Stop() + } + var trigger *time.Timer + trigger = time.AfterFunc(d, func() { + db.mu.Lock() + defer db.mu.Unlock() + if trigger == db.t { + f() + db.t = nil + } + }) + db.t = trigger + } +} diff --git a/modules/util/timer_test.go b/modules/util/timer_test.go new file mode 100644 index 0000000000000..602800c2482c4 --- /dev/null +++ b/modules/util/timer_test.go @@ -0,0 +1,30 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestDebounce(t *testing.T) { + var c int64 + d := Debounce(50 * time.Millisecond) + d(func() { atomic.AddInt64(&c, 1) }) + assert.EqualValues(t, 0, atomic.LoadInt64(&c)) + d(func() { atomic.AddInt64(&c, 1) }) + d(func() { atomic.AddInt64(&c, 1) }) + time.Sleep(100 * time.Millisecond) + assert.EqualValues(t, 1, atomic.LoadInt64(&c)) + d(func() { atomic.AddInt64(&c, 1) }) + assert.EqualValues(t, 1, atomic.LoadInt64(&c)) + d(func() { atomic.AddInt64(&c, 1) }) + d(func() { atomic.AddInt64(&c, 1) }) + d(func() { atomic.AddInt64(&c, 1) }) + time.Sleep(100 * time.Millisecond) + assert.EqualValues(t, 2, atomic.LoadInt64(&c)) +} diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go deleted file mode 100644 index 75d062d7aaaa3..0000000000000 --- a/modules/watcher/watcher.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package watcher - -import ( - "context" - "io/fs" - "os" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - - "github.com/fsnotify/fsnotify" -) - -// CreateWatcherOpts are options to configure the watcher -type CreateWatcherOpts struct { - // PathsCallback is used to set the required paths to watch - PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error - - // BeforeCallback is called before any files are watched - BeforeCallback func() - - // Between Callback is called between after a watched event has occurred - BetweenCallback func() - - // AfterCallback is called as this watcher ends - AfterCallback func() -} - -// CreateWatcher creates a watcher labelled with the provided description and running with the provided options. -// The created watcher will create a subcontext from the provided ctx and register it with the process manager. -func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) { - go run(ctx, desc, opts) -} - -func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { - if opts.BeforeCallback != nil { - opts.BeforeCallback() - } - if opts.AfterCallback != nil { - defer opts.AfterCallback() - } - ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true) - defer finished() - - log.Trace("Watcher loop starting for %s", desc) - defer log.Trace("Watcher loop ended for %s", desc) - - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - return - } - if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error { - if err != nil && !os.IsNotExist(err) { - return err - } - log.Trace("Watcher: %s watching %q", desc, path) - _ = watcher.Add(path) - return nil - }); err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - _ = watcher.Close() - return - } - - // Note we don't call the BetweenCallback here - - for { - select { - case event, ok := <-watcher.Events: - if !ok { - _ = watcher.Close() - return - } - log.Debug("Watched file for %s had event: %v", desc, event) - case err, ok := <-watcher.Errors: - if !ok { - _ = watcher.Close() - return - } - log.Error("Error whilst watching files for %s: %v", desc, err) - case <-ctx.Done(): - _ = watcher.Close() - return - } - - // Recreate the watcher - only call the BetweenCallback after the new watcher is set-up - _ = watcher.Close() - watcher, err = fsnotify.NewWatcher() - if err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - return - } - if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { - if err != nil { - return err - } - _ = watcher.Add(path) - return nil - }); err != nil { - log.Error("Unable to create watcher for %s: %v", desc, err) - _ = watcher.Close() - return - } - - // Inform our BetweenCallback that there has been an event - if opts.BetweenCallback != nil { - opts.BetweenCallback() - } - } -} diff --git a/routers/init.go b/routers/init.go index 8cf53fc108de0..c539975acadcc 100644 --- a/routers/init.go +++ b/routers/init.go @@ -71,13 +71,6 @@ func mustInitCtx(ctx context.Context, fn func(ctx context.Context) error) { } } -// InitGitServices init new services for git, this is also called in `contrib/pr/checkout.go` -func InitGitServices() { - setting.LoadSettings() - mustInit(storage.Init) - mustInit(repo_service.Init) -} - func syncAppConfForGit(ctx context.Context) error { runtimeState := new(system.RuntimeState) if err := system.AppState.Get(runtimeState); err != nil { @@ -172,7 +165,7 @@ func GlobalInitInstalled(ctx context.Context) { mustInit(ssh.Init) auth.Init() - svg.Init() + mustInit(svg.Init) actions_service.Init() diff --git a/routers/install/routes.go b/routers/install/routes.go index 82d9c34b41f18..df82ba2e4c3e7 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -8,7 +8,6 @@ import ( "fmt" "html" "net/http" - "path" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" @@ -89,10 +88,7 @@ func Routes(ctx goctx.Context) *web.Route { r.Use(middle) } - r.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ - Directory: path.Join(setting.StaticRootPath, "public"), - Prefix: public.AssetsURLPathPrefix, - }), "InstallAssetsHandler")) + r.Use(web.WrapWithPrefix("/assets/", public.AssetsHandlerFunc("/assets/"), "AssetsHandler")) r.Use(session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, diff --git a/routers/install/setting.go b/routers/install/setting.go index 68984f1e7837d..dadefa26a2930 100644 --- a/routers/install/setting.go +++ b/routers/install/setting.go @@ -30,7 +30,7 @@ func PreloadSettings(ctx context.Context) bool { } setting.LoadSettingsForInstall() - svg.Init() + _ = svg.Init() } return !setting.InstallLock @@ -47,6 +47,5 @@ func reloadSettings(ctx context.Context) { } else { log.Fatal("ORM engine initialization failed: %v", err) } - svg.Init() } } diff --git a/routers/web/devtest/devtest.go b/routers/web/devtest/devtest.go index eb77d0b927b6b..784940909a612 100644 --- a/routers/web/devtest/devtest.go +++ b/routers/web/devtest/devtest.go @@ -15,15 +15,16 @@ import ( // List all devtest templates, they will be used for e2e tests for the UI components func List(ctx *context.Context) { - templateNames := templates.GetTemplateAssetNames() + templateNames, err := templates.AssetFS().ListFiles("devtest", true) + if err != nil { + ctx.ServerError("AssetFS().ListFiles", err) + return + } var subNames []string - const prefix = "templates/devtest/" for _, tmplName := range templateNames { - if strings.HasPrefix(tmplName, prefix) { - subName := strings.TrimSuffix(strings.TrimPrefix(tmplName, prefix), ".tmpl") - if subName != "list" { - subNames = append(subNames, subName) - } + subName := strings.TrimSuffix(tmplName, ".tmpl") + if subName != "list" { + subNames = append(subNames, subName) } } ctx.Data["SubNames"] = subNames diff --git a/routers/web/web.go b/routers/web/web.go index cee8bafcdba1a..a4a1b7113c4d9 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -103,11 +103,7 @@ func buildAuthGroup() *auth_service.Group { func Routes(ctx gocontext.Context) *web.Route { routes := web.NewRoute() - routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ - Directory: path.Join(setting.StaticRootPath, "public"), - Prefix: public.AssetsURLPathPrefix, - CorsHandler: CorsHandler(), - }), "AssetsHandler")) + routes.Use(web.WrapWithPrefix("/assets/", web.Wrap(CorsHandler(), public.AssetsHandlerFunc("/assets/")), "AssetsHandler")) sessioner := session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, From 58b36cc42291e274dbc80d2d617e63119c64542c Mon Sep 17 00:00:00 2001 From: delvh Date: Wed, 12 Apr 2023 13:59:17 +0200 Subject: [PATCH 06/12] Add tooltips to `Hide comment type` settings where necessary (#21306) Previously, this setting was pretty confusing for users, especially the difference between "reference" and "issue reference". Related: #21321. --- options/locale/locale_en-US.ini | 3 +++ templates/user/settings/appearance.tmpl | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cf3208b5bdcb1..b5e1c718ee982 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -611,6 +611,9 @@ cancel = Cancel language = Language ui = Theme hidden_comment_types = Hidden comment types +hidden_comment_types_description = Comment types checked here will not be shown inside issue pages. Checking "Label" for example removes all " added/removed