From ae33c7059a11ac03ce6f2e5cfce761c2ee170bf6 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Thu, 22 Feb 2024 13:04:59 +0400 Subject: [PATCH 1/2] object: Limit header length to 16KB Previously, NeoFS used 4MB as object header's length limit. The value originally resulted from the default max gRPC message length. Now header length can be up to 16KB only. To ensure the safety of data uploaded before the restriction was introduced, this limit does not apply to intra-container replication. Refs https://github.com/nspcc-dev/neofs-api/issues/262. Signed-off-by: Leonard Lyubich --- CHANGELOG.md | 1 + cmd/neofs-cli/modules/util/acl.go | 1 - go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ pkg/services/object/acl/v2/service.go | 14 +++++++++++++- 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f37debde2a..25bea1b507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Changelog for NeoFS Node ### Changed - IR now checks format of NULL and numeric eACL filters specified in the protocol (#2742) - Empty filter value is now treated as `NOT_PRESENT` op by CLI `acl extended create` cmd (#2742) +- Storage nodes no longer accept objects with header larger than 16KB (#2749) ### Removed - Object notifications incl. NATS (#2750) diff --git a/cmd/neofs-cli/modules/util/acl.go b/cmd/neofs-cli/modules/util/acl.go index 9cc95d6d1e..4ea40dcf4b 100644 --- a/cmd/neofs-cli/modules/util/acl.go +++ b/cmd/neofs-cli/modules/util/acl.go @@ -135,7 +135,6 @@ func eaclFiltersToString(fs []eacl.Filter) string { } _, _ = tw.Write([]byte(f.Key())) - switch f.Matcher() { case eacl.MatchStringEqual: _, _ = tw.Write([]byte("\t==\t")) diff --git a/go.mod b/go.mod index e689c0f387..8db2d5ae0e 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/go-github/v39 v39.2.0 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.17.2 + github.com/klauspost/compress v1.17.6 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.0 @@ -18,7 +18,7 @@ require ( github.com/nspcc-dev/neo-go v0.105.1 github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240213170208-cfca09b5acbe github.com/nspcc-dev/neofs-contract v0.19.1 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240221185518-cbaf23c6aa7a + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240227143352-846741cb85a3 github.com/nspcc-dev/tzhash v1.7.1 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.8.2 @@ -32,8 +32,8 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sync v0.3.0 - golang.org/x/sys v0.15.0 - golang.org/x/term v0.15.0 + golang.org/x/sys v0.17.0 + golang.org/x/term v0.17.0 google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 @@ -94,7 +94,7 @@ require ( github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.5 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.17.0 // indirect + golang.org/x/crypto v0.19.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect diff --git a/go.sum b/go.sum index 1707d9b1ba..f87e44c6f5 100644 --- a/go.sum +++ b/go.sum @@ -159,8 +159,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -186,8 +186,8 @@ github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svo github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -251,8 +251,8 @@ github.com/nspcc-dev/neofs-contract v0.19.1 h1:U1Uh+MlzfkalO0kRJ2pADZyHrmAOroC6K github.com/nspcc-dev/neofs-contract v0.19.1/go.mod h1:ZOGouuwuHpgvYkx/LCGufGncIzEUhYEO18LL4cWEbyw= github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09fZo+KYi4= github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240221185518-cbaf23c6aa7a h1:vmN8Sm8Wna5BrgkGBvt5cnPTzU4Fu0JzC6VnwDNiDIA= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240221185518-cbaf23c6aa7a/go.mod h1:icGhc6HFg+yKivBUoP7cut62SASuijDiWD5Txd6vWqY= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240227143352-846741cb85a3 h1:ADjuSMRdYW4FVtOAaFUi+imycOXu1Ej7yIVK37grDtY= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240227143352-846741cb85a3/go.mod h1:GED7qcgut188OQYY7lak/cGN8DHtjqQbrXdSt/82QNE= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.7.1 h1:6zmexLqdTF/ssbUAh7XJS7RxgKWaw28kdNpE/4UFdEU= @@ -359,8 +359,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -502,11 +502,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 89f0b9f434..33b3ee1c61 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -12,6 +12,7 @@ import ( apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" "github.com/nspcc-dev/neofs-sdk-go/container/acl" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + objectsdk "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" sessionSDK "github.com/nspcc-dev/neofs-sdk-go/session" "github.com/nspcc-dev/neofs-sdk-go/user" @@ -524,15 +525,26 @@ func (p putStreamBasicChecker) Send(request *objectV2.PutRequest) error { return err } + replication := reqInfo.requestRole == acl.RoleContainer && request.GetMetaHeader().GetTTL() == 1 if tombstone { // the only exception when writing tombstone should not be treated as deletion // is intra-container replication: container nodes must be able to replicate // such objects while deleting is prohibited - if reqInfo.requestRole == acl.RoleContainer && request.GetMetaHeader().GetTTL() == 1 { + if replication { reqInfo.operation = acl.OpObjectPut } } + if !replication { + // header length is unchecked for replication because introducing a restriction + // should not prevent the replication of objects created before. + // See also https://github.com/nspcc-dev/neofs-api/issues/293 + hdrLen := part.GetHeader().StableSize() + if hdrLen > objectsdk.MaxHeaderLen { + return fmt.Errorf("object header length exceeds the limit: %d>%d", hdrLen, objectsdk.MaxHeaderLen) + } + } + reqInfo.obj = obj if !p.source.checker.CheckBasicACL(reqInfo) || !p.source.checker.StickyBitCheck(reqInfo, idOwner) { From a6343cf1964deb2d9d2b714a30c8ce1d3054851b Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 28 Feb 2024 11:25:33 +0400 Subject: [PATCH 2/2] cli: Drop empty case in switch statement Signed-off-by: Leonard Lyubich --- cmd/neofs-cli/modules/util/acl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/neofs-cli/modules/util/acl.go b/cmd/neofs-cli/modules/util/acl.go index 4ea40dcf4b..360a1a0043 100644 --- a/cmd/neofs-cli/modules/util/acl.go +++ b/cmd/neofs-cli/modules/util/acl.go @@ -135,6 +135,7 @@ func eaclFiltersToString(fs []eacl.Filter) string { } _, _ = tw.Write([]byte(f.Key())) + //nolint:exhaustive switch f.Matcher() { case eacl.MatchStringEqual: _, _ = tw.Write([]byte("\t==\t")) @@ -150,7 +151,6 @@ func eaclFiltersToString(fs []eacl.Filter) string { _, _ = tw.Write([]byte("\t<=\t")) case eacl.MatchNotPresent: _, _ = tw.Write([]byte("\tNULL\t")) - case eacl.MatchUnknown: } _, _ = tw.Write([]byte(f.Value() + "\t"))