From 538bf8e77ea28413ebc5e562adf3fd5a41b3241f Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 30 Mar 2023 18:54:47 +0800 Subject: [PATCH 1/2] fix --- build/update-locales.sh | 15 ++------ modules/translation/i18n/i18n_test.go | 53 +++++++++++++++++++++++++++ options/locale/locale_en-US.ini | 8 ++-- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/build/update-locales.sh b/build/update-locales.sh index 596ddfec032b..6f9ee334be4a 100755 --- a/build/update-locales.sh +++ b/build/update-locales.sh @@ -17,17 +17,10 @@ fi mv ./options/locale/locale_en-US.ini ./options/ -# the "ini" library for locale has many quirks -# * `a="xx"` gets `xx` (no quote) -# * `a=x\"y` gets `x\"y` (no unescaping) -# * `a="x\"y"` gets `"x\"y"` (no unescaping, the quotes are still there) -# * `a='x\"y'` gets `x\"y` (no unescaping, no quote) -# * `a="foo` gets `"foo` (although the quote is not closed) -# * 'a=`foo`' works like single-quote -# crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes -# crowdin always outputs quoted strings if there are quotes in the strings. - -# this script helps to unquote the crowdin outputs for the quirky ini library +# the "ini" library for locale has many quirks, its behavior is different from Crowdin. +# see i18n_test.go for more details + +# this script helps to unquote the Crowdin outputs for the quirky ini library # * find all `key="...\"..."` lines # * remove the leading quote # * remove the trailing quote diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go index 76a91522bad2..5be5303d9c70 100644 --- a/modules/translation/i18n/i18n_test.go +++ b/modules/translation/i18n/i18n_test.go @@ -4,6 +4,7 @@ package i18n import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -75,3 +76,55 @@ c=22 assert.Equal(t, "21", ls.Tr("lang1", "b")) assert.Equal(t, "22", ls.Tr("lang1", "c")) } + +func TestLocaleStoreQuirks(t *testing.T) { + const nl = "\n" + q := func(q1, s string, q2 ...string) string { + return q1 + s + strings.Join(q2, "") + } + testDataList := []struct { + in string + out string + hint string + }{ + {` xx`, `xx`, "simple, no quote"}, + {`" xx"`, ` xx`, "simple, double-quote"}, + {`' xx'`, ` xx`, "simple, single-quote"}, + {"` xx`", ` xx`, "simple, back-quote"}, + + {`x\"y`, `x\"y`, "no unescape, simple"}, + {q(`"`, `x\"y`, `"`), `"x\"y"`, "unescape, double-quote"}, + {q(`'`, `x\"y`, `'`), `x\"y`, "no unescape, single-quote"}, + {q("`", `x\"y`, "`"), `x\"y`, "no unescape, back-quote"}, + + {q(`"`, `x\"y`) + nl + "b=", `"x\"y`, "half open, double-quote"}, + {q(`'`, `x\"y`) + nl + "b=", `'x\"y`, "half open, single-quote"}, + {q("`", `x\"y`) + nl + "b=`", `x\"y` + nl + "b=", "half open, back-quote, multi-line"}, + + {`x ; y`, `x ; y`, "inline comment (;)"}, + {`x # y`, `x # y`, "inline comment (#)"}, + {`x \; y`, `x ; y`, `inline comment (\;)`}, + {`x \# y`, `x # y`, `inline comment (\#)`}, + } + + for _, testData := range testDataList { + ls := NewLocaleStore() + err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil) + assert.NoError(t, err, testData.hint) + assert.Equal(t, testData.out, ls.Tr("lang1", "a"), testData.hint) + } + + // TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes + // and Crowdin always outputs quoted strings if there are quotes in the strings. + // So, Gitea's `key="quoted" unquoted` content shouldn't be used on Crowdin directly, + // it should be converted to `key="\"quoted\" unquoted"` first. + // TODO: We can not use UnescapeValueDoubleQuotes=true, because there are a lot of back-quotes in en-US.ini, + // then Crowdin will output: + // > key = "`x \" y`" + // Then Gitea will read a string with back-quotes, which is incorrect. + // TODO: Crowdin might generate multi-line strings, quoted by double-quote, it's not supported by LocaleStore + // LocaleStore uses back-quote for multi-line strings, it's not supported by Crowdin. + // TODO: Crowdin doesn't support back-quote as string quoter, it mainly uses double-quote + // so, the following line will be parsed as: value="`first", comment="second`" on Crowdin + // > a = `first; second` +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3f47af826e54..d2ed3ef4fb40 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2140,10 +2140,10 @@ settings.dismiss_stale_approvals_desc = When new commits that change the content settings.require_signed_commits = Require Signed Commits settings.require_signed_commits_desc = Reject pushes to this branch if they are unsigned or unverifiable. settings.protect_branch_name_pattern = Protected Branch Name Pattern -settings.protect_protected_file_patterns = `Protected file patterns (separated using semicolon ';'):` -settings.protect_protected_file_patterns_desc = `Protected files are not allowed to be changed directly even if user has rights to add, edit, or delete files in this branch. Multiple patterns can be separated using semicolon (';'). See github.com/gobwas/glob documentation for pattern syntax. Examples: .drone.yml, /docs/**/*.txt.` -settings.protect_unprotected_file_patterns = `Unprotected file patterns (separated using semicolon ';'):` -settings.protect_unprotected_file_patterns_desc = `Unprotected files that are allowed to be changed directly if user has write access, bypassing push restriction. Multiple patterns can be separated using semicolon (';'). See github.com/gobwas/glob documentation for pattern syntax. Examples: .drone.yml, /docs/**/*.txt.` +settings.protect_protected_file_patterns = "Protected file patterns (separated using semicolon ';'):" +settings.protect_protected_file_patterns_desc = "Protected files are not allowed to be changed directly even if user has rights to add, edit, or delete files in this branch. Multiple patterns can be separated using semicolon (';'). See github.com/gobwas/glob documentation for pattern syntax. Examples: .drone.yml, /docs/**/*.txt." +settings.protect_unprotected_file_patterns = "Unprotected file patterns (separated using semicolon ';'):" +settings.protect_unprotected_file_patterns_desc = "Unprotected files that are allowed to be changed directly if user has write access, bypassing push restriction. Multiple patterns can be separated using semicolon (';'). See github.com/gobwas/glob documentation for pattern syntax. Examples: .drone.yml, /docs/**/*.txt." settings.add_protected_branch = Enable protection settings.delete_protected_branch = Disable protection settings.update_protect_branch_success = Branch protection for rule '%s' has been updated. From 649bca87a594d3fcd88343d97b5f0c113d972de1 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 30 Mar 2023 20:14:35 +0800 Subject: [PATCH 2/2] fine tune tests --- modules/translation/i18n/i18n_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go index 5be5303d9c70..085d03811f6d 100644 --- a/modules/translation/i18n/i18n_test.go +++ b/modules/translation/i18n/i18n_test.go @@ -112,6 +112,7 @@ func TestLocaleStoreQuirks(t *testing.T) { err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil) assert.NoError(t, err, testData.hint) assert.Equal(t, testData.out, ls.Tr("lang1", "a"), testData.hint) + assert.NoError(t, ls.Close()) } // TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes