From 9f58dce13b8d107689fba6a473af0f3da9864dae Mon Sep 17 00:00:00 2001 From: Matt Jibson Date: Wed, 8 Aug 2018 16:41:42 -0400 Subject: [PATCH] storageccl: improve support for non-S3 endpoints Remove the requirement for a region if an endpoint is specified by using a bogus default region string. If an endpoint is specified, use the old style S3 urls (http://endpoint/bucket instead of http://bucket.endpoint/). These two changes were tested with Minio and DO Spaces and both now work as expected (without specifying a region and no weird endpoint URL). Fixes #24224 Fixes #21412 Release note (sql change): Improve support for S3-compatible endpoints in BACKUP, RESTORE, and IMPORT. The AWS_REGION parameter is no longer required. Services like Digital Ocean Spaces and Minio now work correctly. --- pkg/ccl/storageccl/export_storage.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/ccl/storageccl/export_storage.go b/pkg/ccl/storageccl/export_storage.go index 021ed6207030..7bfa209d9df0 100644 --- a/pkg/ccl/storageccl/export_storage.go +++ b/pkg/ccl/storageccl/export_storage.go @@ -529,7 +529,7 @@ func makeS3Storage( if conf.Endpoint != "" { config.Endpoint = &conf.Endpoint if conf.Region == "" { - return nil, errors.New("s3 region must be specified when using custom endpoints") + region = "default-region" } client, err := makeHTTPClient(settings) if err != nil { @@ -552,6 +552,9 @@ func makeS3Storage( } } sess.Config.Region = aws.String(region) + if conf.Endpoint != "" { + sess.Config.S3ForcePathStyle = aws.Bool(true) + } return &s3Storage{ bucket: aws.String(conf.Bucket), conf: conf,