diff --git a/.travis.yml b/.travis.yml index c9844e9..b0474b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,9 @@ go: - 1.7 - 1.8 - 1.9 - - '1.10' - - '1.11' + - '1.10.x' + - '1.11.x' + - '1.12.x' - master sudo: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 658f348..942516c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog +## [0.11.1] (2019-xx-yy) + +### Bugfix + +* 修复当 url 或 headers 相关参数的值中包含空格时会出现服务端返回签名不匹配的问题。(via [#13]) + + ## [0.11.0] (2018-12-08) ### 不兼容旧版的变更 @@ -132,6 +139,7 @@ * 完成大部分 Bucket API(还剩一个 Put Bucket Lifecycle) +[0.11.1]: https://github.com/mozillazg/go-cos/compare/v0.11.0...v0.11.1 [0.11.0]: https://github.com/mozillazg/go-cos/compare/v0.10.0...v0.11.0 [0.10.0]: https://github.com/mozillazg/go-cos/compare/v0.9.0...v0.10.0 [0.9.0]: https://github.com/mozillazg/go-cos/compare/v0.8.0...v0.9.0 @@ -147,3 +155,4 @@ [7dcd701]: https://github.com/mozillazg/go-cos/commit/7dcd701975f483d57525b292ab31d0f9a6c8866c [#7]: https://github.com/mozillazg/go-cos/pull/7 [@jojohappy]: https://github.com/jojohappy +[#13]: https://github.com/mozillazg/go-cos/pull/13 diff --git a/_example/bucket/get.go b/_example/bucket/get.go index 37483dd..a4408ca 100644 --- a/_example/bucket/get.go +++ b/_example/bucket/get.go @@ -30,12 +30,21 @@ func main() { Prefix: "test", MaxKeys: 3, } - v, _, err := c.Bucket.Get(context.Background(), opt) + v, resp, err := c.Bucket.Get(context.Background(), opt) if err != nil { panic(err) } + resp.Body.Close() for _, c := range v.Contents { fmt.Printf("%s, %d\n", c.Key, c.Size) } + + // 测试特殊字符 + opt.Prefix = "test/put_ + !'()* option" + _, resp, err = c.Bucket.Get(context.Background(), opt) + if err != nil { + panic(err) + } + resp.Body.Close() } diff --git a/_example/object/put.go b/_example/object/put.go index a22f456..678b718 100644 --- a/_example/object/put.go +++ b/_example/object/put.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "net/http" "os" "strings" @@ -33,19 +34,33 @@ func main() { panic(err) } - name = "test/put_option.go" + // 测试上传以及特殊字符 + name = "test/put_ + !'()* option.go" + contentDisposition := "attachment; filename=Hello - world!(+)'*.go" f = strings.NewReader("test xxx") opt := &cos.ObjectPutOptions{ ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ - ContentType: "text/html", + ContentType: "text/html", + ContentDisposition: contentDisposition, }, ACLHeaderOptions: &cos.ACLHeaderOptions{ - //XCosACL: "public-read", + // XCosACL: "public-read", XCosACL: "private", }, } - _, err = c.Object.Put(context.Background(), name, f, opt) + resp, err := c.Object.Put(context.Background(), name, f, opt) if err != nil { panic(err) } + resp.Body.Close() + + // 测试特殊字符 + resp, err = c.Object.Get(context.Background(), name, nil) + if err != nil { + panic(err) + } + resp.Body.Close() + if resp.Header.Get("Content-Disposition") != contentDisposition { + panic(errors.New("wong Content-Disposition")) + } } diff --git a/auth.go b/auth.go index 3390d31..7966d61 100644 --- a/auth.go +++ b/auth.go @@ -85,6 +85,8 @@ func (a *AuthTime) keyString() string { } // newAuthorization 通过一系列步骤生成最终需要的 Authorization 字符串 +// +// https://cloud.tencent.com/document/product/436/7778 func newAuthorization(auth Auth, req *http.Request, authTime AuthTime) string { secretKey := auth.SecretKey secretID := auth.SecretID @@ -156,17 +158,59 @@ func genFormatString(method string, uri url.URL, formatParameters, formatHeaders ) } +// https://github.com/tencentyun/cos-nodejs-sdk-v5/blob/a1dad3e9e3776cd24c97975f3aa47631e5001ff0/sdk/util.js#L11 +func camSafeURLEncode(s string) string { + s = encodeURIComponent(s) + s = strings.Replace(s, "!", "%21", -1) + s = strings.Replace(s, "'", "%27", -1) + s = strings.Replace(s, "(", "%28", -1) + s = strings.Replace(s, ")", "%29", -1) + s = strings.Replace(s, "*", "%2A", -1) + return s +} + +type valuesForSign map[string][]string + +func (vs valuesForSign) Add(key, value string) { + key = strings.ToLower(key) + vs[key] = append(vs[key], value) +} + +// https://cloud.tencent.com/document/product/436/7778 +// https://github.com/tencentyun/cos-nodejs-sdk-v5/blob/a1dad3e9e3776cd24c97975f3aa47631e5001ff0/sdk/util.js#L42-L69 +func (vs valuesForSign) Encode() string { + var keys []string + for k := range vs { + keys = append(keys, k) + } + // 字典序排序 + sort.Strings(keys) + + var pairs []string + for _, k := range keys { + items := vs[k] + sort.Strings(items) + for _, v := range items { + pairs = append( + pairs, + fmt.Sprintf("%s=%s", camSafeURLEncode(k), camSafeURLEncode(v))) + } + } + // =&= + return strings.Join(pairs, "&") +} + // genFormatParameters 生成 FormatParameters 和 SignedParameterList func genFormatParameters(parameters url.Values) (formatParameters string, signedParameterList []string) { - ps := url.Values{} + ps := valuesForSign{} for key, values := range parameters { + key = strings.ToLower(key) for _, value := range values { - key = strings.ToLower(key) ps.Add(key, value) signedParameterList = append(signedParameterList, key) } } - //formatParameters = strings.ToLower(ps.Encode()) + formatParameters = ps.Encode() sort.Strings(signedParameterList) return @@ -174,16 +218,17 @@ func genFormatParameters(parameters url.Values) (formatParameters string, signed // genFormatHeaders 生成 FormatHeaders 和 SignedHeaderList func genFormatHeaders(headers http.Header) (formatHeaders string, signedHeaderList []string) { - hs := url.Values{} + hs := valuesForSign{} for key, values := range headers { + key = strings.ToLower(key) for _, value := range values { - key = strings.ToLower(key) if isSignHeader(key) { hs.Add(key, value) signedHeaderList = append(signedHeaderList, key) } } } + formatHeaders = hs.Encode() sort.Strings(signedHeaderList) return diff --git a/auth_test.go b/auth_test.go index 1bb5df1..311bd68 100644 --- a/auth_test.go +++ b/auth_test.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/http" + "net/url" + "reflect" "testing" "time" ) @@ -104,3 +106,209 @@ func (t *testingTransport) RoundTrip(req *http.Request) (*http.Response, error) t.called++ return http.DefaultTransport.RoundTrip(req) } + +func Test_camSafeURLEncode(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "no replace", + args: args{"1234 +abc0AB#@"}, + want: "1234%20%2Babc0AB%23%40", + }, + { + name: "replace", + args: args{"1234 +abc0AB#@,!'()*"}, + want: "1234%20%2Babc0AB%23%40%2C%21%27%28%29%2A", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := camSafeURLEncode(tt.args.s); got != tt.want { + t.Errorf("camSafeURLEncode() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_valuesForSign_Encode(t *testing.T) { + tests := []struct { + name string + vs valuesForSign + want string + }{ + { + name: "test escape", + vs: valuesForSign{ + "test+233": {"value 666"}, + "test+234": {"value 667"}, + }, + want: "test%2B233=value%20666&test%2B234=value%20667", + }, + { + name: "test order", + vs: valuesForSign{ + "test_233": {"value_666"}, + "233": {"value_2"}, + "test_666": {"value_123"}, + }, + want: "233=value_2&test_233=value_666&test_666=value_123", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.vs.Encode(); got != tt.want { + t.Errorf("valuesForSign.Encode() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_valuesForSign_Add(t *testing.T) { + type args struct { + key string + value string + } + tests := []struct { + name string + vs valuesForSign + args args + want valuesForSign + }{ + { + name: "add new key", + vs: valuesForSign{}, + args: args{"test_key", "value_233"}, + want: valuesForSign{"test_key": {"value_233"}}, + }, + { + name: "extend key", + vs: valuesForSign{"test_key": {"value_233"}}, + args: args{"test_key", "value_666"}, + want: valuesForSign{"test_key": {"value_233", "value_666"}}, + }, + { + name: "key to lower(add)", + vs: valuesForSign{}, + args: args{"TEST_KEY", "value_233"}, + want: valuesForSign{"test_key": {"value_233"}}, + }, + { + name: "key to lower(extend)", + vs: valuesForSign{"test_key": {"value_233"}}, + args: args{"TEST_KEY", "value_666"}, + want: valuesForSign{"test_key": {"value_233", "value_666"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.vs.Add(tt.args.key, tt.args.value) + if !reflect.DeepEqual(tt.vs, tt.want) { + t.Errorf("%v, want %v", tt.vs, tt.want) + } + }) + } +} + +func Test_genFormatParameters(t *testing.T) { + type args struct { + parameters url.Values + } + tests := []struct { + name string + args args + wantFormatParameters string + wantSignedParameterList []string + }{ + { + name: "test order", + args: args{url.Values{ + "test_key_233": {"666"}, + "233": {"222"}, + "test_key_2": {"value"}, + }}, + wantFormatParameters: "233=222&test_key_2=value&test_key_233=666", + wantSignedParameterList: []string{"233", "test_key_2", "test_key_233"}, + }, + { + name: "test escape", + args: args{url.Values{ + "Test+key": {"666 value"}, + "233 666": {"22+2"}, + }}, + wantFormatParameters: "233%20666=22%2B2&test%2Bkey=666%20value", + wantSignedParameterList: []string{"233 666", "test+key"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFormatParameters, gotSignedParameterList := genFormatParameters(tt.args.parameters) + if gotFormatParameters != tt.wantFormatParameters { + t.Errorf("genFormatParameters() gotFormatParameters = %v, want %v", gotFormatParameters, tt.wantFormatParameters) + } + if !reflect.DeepEqual(gotSignedParameterList, tt.wantSignedParameterList) { + t.Errorf("genFormatParameters() gotSignedParameterList = %v, want %v", gotSignedParameterList, tt.wantSignedParameterList) + } + }) + } +} + +func Test_genFormatHeaders(t *testing.T) { + type args struct { + headers http.Header + } + tests := []struct { + name string + args args + wantFormatHeaders string + wantSignedHeaderList []string + }{ + { + name: "test order", + args: args{http.Header{ + "host": {"example.com"}, + "content-length": {"22"}, + "content-md5": {"xxx222"}, + }}, + wantFormatHeaders: "content-length=22&content-md5=xxx222&host=example.com", + wantSignedHeaderList: []string{"content-length", "content-md5", "host"}, + }, + { + name: "test escape", + args: args{http.Header{ + "host": {"example.com"}, + "content-length": {"22"}, + "Content-Disposition": {"attachment; filename=hello - world!(+).go"}, + }}, + wantFormatHeaders: "content-disposition=attachment%3B%20filename%3Dhello%20-%20world%21%28%2B%29.go&content-length=22&host=example.com", + wantSignedHeaderList: []string{"content-disposition", "content-length", "host"}, + }, + { + name: "test skip key", + args: args{http.Header{ + "Host": {"example.com"}, + "content-length": {"22"}, + "x-cos-xyz": {"lala"}, + "Content-Type": {"text/html"}, + }}, + wantFormatHeaders: "content-length=22&host=example.com&x-cos-xyz=lala", + wantSignedHeaderList: []string{"content-length", "host", "x-cos-xyz"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFormatHeaders, gotSignedHeaderList := genFormatHeaders(tt.args.headers) + if gotFormatHeaders != tt.wantFormatHeaders { + t.Errorf("genFormatHeaders() gotFormatHeaders = %v, want %v", gotFormatHeaders, tt.wantFormatHeaders) + } + if !reflect.DeepEqual(gotSignedHeaderList, tt.wantSignedHeaderList) { + t.Errorf("genFormatHeaders() gotSignedHeaderList = %v, want %v", gotSignedHeaderList, tt.wantSignedHeaderList) + } + }) + } +} diff --git a/helper.go b/helper.go index d947ede..08c5a35 100644 --- a/helper.go +++ b/helper.go @@ -73,7 +73,7 @@ func encodeURIComponent(s string) string { } b.WriteString(s[written:i]) - fmt.Fprintf(&b, "%%%02x", c) + fmt.Fprintf(&b, "%%%02X", c) written = i + 1 } diff --git a/helper_test.go b/helper_test.go index 75ef294..9cb4c16 100644 --- a/helper_test.go +++ b/helper_test.go @@ -22,3 +22,42 @@ func Test_calMD5Digest(t *testing.T) { t.Errorf("calMD5Digest request md5: %+v, want %+v", got, want) } } + +func Test_encodeURIComponent(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "empty", + args: args{""}, + want: "", + }, + { + name: "no escape", + args: args{"0123456789abcdefghijkhlmnopqrstuvwxyzABCDEFGHIJKHLMNOPQRSTUVWXYZ-_.!~*'()"}, + want: "0123456789abcdefghijkhlmnopqrstuvwxyzABCDEFGHIJKHLMNOPQRSTUVWXYZ-_.!~*'()", + }, + { + name: "escape", + args: args{"+ $@#/"}, + want: "%2B%20%24%40%23%2F", + }, + { + name: "escape+no", + args: args{"+ $abc@#13/0"}, + want: "%2B%20%24abc%40%2313%2F0", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := encodeURIComponent(tt.args.s); got != tt.want { + t.Errorf("encodeURIComponent() = %v, want %v", got, tt.want) + } + }) + } +}