Skip to content

Commit

Permalink
Trying to speed up tests a bit.
Browse files Browse the repository at this point in the history
Signed-off-by: Bartek Plotka <[email protected]>
  • Loading branch information
bwplotka committed Oct 22, 2019
1 parent f4ec30e commit 6d61779
Show file tree
Hide file tree
Showing 7 changed files with 66 additions and 80 deletions.
2 changes: 1 addition & 1 deletion pkg/objstore/azure/azure.go
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err
conf := &Config{
StorageAccountName: os.Getenv("AZURE_STORAGE_ACCOUNT"),
StorageAccountKey: os.Getenv("AZURE_STORAGE_ACCESS_KEY"),
ContainerName: "thanos-e2e-test",
ContainerName: objstore.CreateTemporaryTestBucketName(t),
}

bc, err := yaml.Marshal(conf)
Expand Down
19 changes: 5 additions & 14 deletions pkg/objstore/cos/cos.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,17 @@ import (
"context"
"fmt"
"io"
"math/rand"
"net/http"
"os"
"strings"
"testing"
"time"

"github.com/go-kit/kit/log"
cos "github.com/mozillazg/go-cos"
"github.com/mozillazg/go-cos"
"github.com/pkg/errors"
"github.com/thanos-io/thanos/pkg/objstore"
"github.com/thanos-io/thanos/pkg/runutil"
yaml "gopkg.in/yaml.v2"
"gopkg.in/yaml.v2"
)

// DirDelim is the delimiter used to model a directory structure in an object store bucket.
Expand Down Expand Up @@ -311,14 +309,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
t.Log("WARNING. Reusing", c.Bucket, "COS bucket for COS tests. Manual cleanup afterwards is required")
return b, func() {}, nil
}

src := rand.NewSource(time.Now().UnixNano())

tmpBucketName := strings.Replace(fmt.Sprintf("test_%x", src.Int63()), "_", "-", -1)
if len(tmpBucketName) >= 31 {
tmpBucketName = tmpBucketName[:31]
}
c.Bucket = tmpBucketName
c.Bucket = objstore.CreateTemporaryTestBucketName(t)

bc, err := yaml.Marshal(c)
if err != nil {
Expand All @@ -333,12 +324,12 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
if _, err := b.client.Bucket.Put(context.Background(), nil); err != nil {
return nil, nil, err
}
t.Log("created temporary COS bucket for COS tests with name", tmpBucketName)
t.Log("created temporary COS bucket for COS tests with name", c.Bucket)

return b, func() {
objstore.EmptyBucket(t, context.Background(), b)
if _, err := b.client.Bucket.Delete(context.Background()); err != nil {
t.Logf("deleting bucket %s failed: %s", tmpBucketName, err)
t.Logf("deleting bucket %s failed: %s", c.Bucket, err)
}
}, nil
}
Expand Down
7 changes: 2 additions & 5 deletions pkg/objstore/gcs/gcs.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,9 @@ import (
"context"
"fmt"
"io"
"math/rand"
"runtime"
"strings"
"testing"
"time"

"cloud.google.com/go/storage"
"github.com/go-kit/kit/log"
Expand All @@ -19,7 +17,7 @@ import (
"golang.org/x/oauth2/google"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
yaml "gopkg.in/yaml.v2"
"gopkg.in/yaml.v2"
)

// DirDelim is the delimiter used to model a directory structure in an object store bucket.
Expand Down Expand Up @@ -169,9 +167,8 @@ func (b *Bucket) Close() error {
func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
src := rand.NewSource(time.Now().UnixNano())
gTestConfig := Config{
Bucket: fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()),
Bucket: objstore.CreateTemporaryTestBucketName(t),
}

bc, err := yaml.Marshal(gTestConfig)
Expand Down
14 changes: 14 additions & 0 deletions pkg/objstore/objstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@ package objstore

import (
"context"
"fmt"
"io"
"math/rand"
"os"
"path/filepath"
"strings"
"testing"
"time"

"github.com/go-kit/kit/log"
Expand Down Expand Up @@ -372,3 +375,14 @@ func (rc *timingReadCloser) Read(b []byte) (n int, err error) {
}
return n, err
}

func CreateTemporaryTestBucketName(t testing.TB) string {
src := rand.NewSource(time.Now().UnixNano())

// Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html.
name := strings.Replace(strings.Replace(fmt.Sprintf("test_%x_%s", src.Int63(), strings.ToLower(t.Name())), "_", "-", -1), "/", "-", -1)
if len(name) >= 63 {
name = name[:63]
}
return name
}
80 changes: 39 additions & 41 deletions pkg/objstore/objtesting/foreach.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ package objtesting
import (
"os"
"testing"
"time"

"github.com/fortytw2/leaktest"
"github.com/thanos-io/thanos/pkg/objstore"
"github.com/thanos-io/thanos/pkg/objstore/azure"
"github.com/thanos-io/thanos/pkg/objstore/cos"
Expand All @@ -21,98 +19,98 @@ import (
// that deletes it after test was run.
// Use THANOS_SKIP_<objstorename>_TESTS to skip explicitly certain tests.
func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) {
t.Parallel()

// Mandatory Inmem.
if ok := t.Run("inmem", func(t *testing.T) {
defer leaktest.CheckTimeout(t, 10*time.Second)()

t.Parallel()
testFn(t, inmem.NewBucket())

}); !ok {
return
}

// Optional GCS.
if _, ok := os.LookupEnv("THANOS_SKIP_GCS_TESTS"); !ok {
bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
testutil.Ok(t, err)
t.Run("gcs", func(t *testing.T) {
bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
testutil.Ok(t, err)

t.Parallel()
defer closeFn()

ok := t.Run("gcs", func(t *testing.T) {
// TODO(bwplotka): Add leaktest when https://github.com/GoogleCloudPlatform/google-cloud-go/issues/1025 is resolved.
testFn(t, bkt)
})
closeFn()
if !ok {
return
}

} else {
t.Log("THANOS_SKIP_GCS_TESTS envvar present. Skipping test against GCS.")
}

// Optional S3.
if _, ok := os.LookupEnv("THANOS_SKIP_S3_AWS_TESTS"); !ok {
// TODO(bwplotka): Allow taking location from envvar.
bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
testutil.Ok(t, err)
t.Run("aws s3", func(t *testing.T) {
// TODO(bwplotka): Allow taking location from envvar.
bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
testutil.Ok(t, err)

t.Parallel()
defer closeFn()

ok := t.Run("aws s3", func(t *testing.T) {
// TODO(bwplotka): Add leaktest when we fix potential leak in minio library.
// We cannot use leaktest for detecting our own potential leaks, when leaktest detects leaks in minio itself.
// This needs to be investigated more.

testFn(t, bkt)
})
closeFn()
if !ok {
return
}

} else {
t.Log("THANOS_SKIP_S3_AWS_TESTS envvar present. Skipping test against S3 AWS.")
}

// Optional Azure.
if _, ok := os.LookupEnv("THANOS_SKIP_AZURE_TESTS"); !ok {
bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
testutil.Ok(t, err)
t.Run("azure", func(t *testing.T) {
bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
testutil.Ok(t, err)

t.Parallel()
defer closeFn()

ok := t.Run("azure", func(t *testing.T) {
testFn(t, bkt)
})
closeFn()
if !ok {
return
}

} else {
t.Log("THANOS_SKIP_AZURE_TESTS envvar present. Skipping test against Azure.")
}

// Optional SWIFT.
if _, ok := os.LookupEnv("THANOS_SKIP_SWIFT_TESTS"); !ok {
container, closeFn, err := swift.NewTestContainer(t)
testutil.Ok(t, err)
t.Run("swift", func(t *testing.T) {
container, closeFn, err := swift.NewTestContainer(t)
testutil.Ok(t, err)

t.Parallel()
defer closeFn()

ok := t.Run("swift", func(t *testing.T) {
testFn(t, container)
})
closeFn()
if !ok {
return
}

} else {
t.Log("THANOS_SKIP_SWIFT_TESTS envvar present. Skipping test against swift.")
}

// Optional COS.
if _, ok := os.LookupEnv("THANOS_SKIP_TENCENT_COS_TESTS"); !ok {
bkt, closeFn, err := cos.NewTestBucket(t)
testutil.Ok(t, err)
t.Run("Tencent cos", func(t *testing.T) {
bkt, closeFn, err := cos.NewTestBucket(t)
testutil.Ok(t, err)

t.Parallel()
defer closeFn()

ok := t.Run("Tencent cos", func(t *testing.T) {
testFn(t, bkt)
})
closeFn()
if !ok {
return
}

} else {
t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.")
}
Expand Down
13 changes: 3 additions & 10 deletions pkg/objstore/s3/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"crypto/tls"
"fmt"
"io"
"math/rand"
"net"
"net/http"
"os"
Expand All @@ -18,15 +17,15 @@ import (

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
minio "github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v6/pkg/credentials"
"github.com/minio/minio-go/v6/pkg/encrypt"
"github.com/pkg/errors"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
"github.com/thanos-io/thanos/pkg/objstore"
"github.com/thanos-io/thanos/pkg/runutil"
yaml "gopkg.in/yaml.v2"
"gopkg.in/yaml.v2"
)

// DirDelim is the delimiter used to model a directory structure in an object store bucket.
Expand Down Expand Up @@ -403,13 +402,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke
}

if c.Bucket == "" {
src := rand.NewSource(time.Now().UnixNano())

// Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html.
bktToCreate = strings.Replace(fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), "_", "-", -1)
if len(bktToCreate) >= 63 {
bktToCreate = bktToCreate[:63]
}
bktToCreate = objstore.CreateTemporaryTestBucketName(t)
}

if err := b.client.MakeBucket(bktToCreate, location); err != nil {
Expand Down
11 changes: 2 additions & 9 deletions pkg/objstore/swift/swift.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,9 @@ import (
"context"
"fmt"
"io"
"math/rand"
"os"
"strings"
"testing"
"time"

"github.com/go-kit/kit/log"
"github.com/gophercloud/gophercloud"
Expand All @@ -19,7 +17,7 @@ import (
"github.com/gophercloud/gophercloud/pagination"
"github.com/pkg/errors"
"github.com/thanos-io/thanos/pkg/objstore"
yaml "gopkg.in/yaml.v2"
"gopkg.in/yaml.v2"
)

// DirDelim is the delimiter used to model a directory structure in an object store bucket.
Expand Down Expand Up @@ -291,12 +289,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) {
return c, func() {}, nil
}

src := rand.NewSource(time.Now().UnixNano())

tmpContainerName := fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63())
if len(tmpContainerName) >= 63 {
tmpContainerName = tmpContainerName[:63]
}
tmpContainerName := objstore.CreateTemporaryTestBucketName(t)

if err := c.createContainer(tmpContainerName); err != nil {
return nil, nil, err
Expand Down

0 comments on commit 6d61779

Please sign in to comment.