Skip to content

Commit

Permalink
Merge #56127
Browse files Browse the repository at this point in the history
56127: importccl: Bazelify importccl test r=miretskiy a=miretskiy

Make importccl test hermetic by using correct directories
and avoiding test data regeneration.

Use 16 shards for the test to speed it up.

Release Notes: None

Co-authored-by: Yevgeniy Miretskiy <[email protected]>
  • Loading branch information
craig[bot] and Yevgeniy Miretskiy committed Nov 3, 2020
2 parents e25ee7a + 2d1bd59 commit 3c9e7b6
Show file tree
Hide file tree
Showing 6 changed files with 136 additions and 33 deletions.
3 changes: 3 additions & 0 deletions pkg/ccl/importccl/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ go_test(
],
data = glob(["testdata/**"]),
embed = [":importccl"],
# It's a large test, so use 16 shards to run test cases.
# See https://docs.bazel.build/versions/master/be/common-definitions.html#test.shard_count
shard_count = 16,
deps = [
"//pkg/base",
"//pkg/blobs",
Expand Down
2 changes: 1 addition & 1 deletion pkg/ccl/importccl/bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func BenchmarkImportWorkload(b *testing.B) {
skip.WithIssue(b, 41932, "broken due to adding keys out-of-order to an sstable")
skip.UnderShort(b, "skipping long benchmark")

dir, cleanup := testutils.TempDir(b)
dir, cleanup := testutils.TestTempDir(b)
defer cleanup()

g := tpcc.FromWarehouses(1)
Expand Down
42 changes: 10 additions & 32 deletions pkg/ccl/importccl/import_stmt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
Expand Down Expand Up @@ -1166,7 +1165,7 @@ func TestImportUserDefinedTypes(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)
ctx := context.Background()
baseDir, cleanup := testutils.TempDir(t)
baseDir, cleanup := testutils.TestTempDir(t)
defer cleanup()
tc := testcluster.StartTestCluster(
t, 1, base.TestClusterArgs{ServerArgs: base.TestServerArgs{ExternalIODir: baseDir}})
Expand Down Expand Up @@ -1265,17 +1264,19 @@ func TestImportUserDefinedTypes(t *testing.T) {
},
}

// Set up a directory for the data files.
err := os.Mkdir(filepath.Join(baseDir, "test"), 0777)
require.NoError(t, err)
// Test IMPORT INTO.
for _, test := range tests {
// Write the test data into a file.
err := ioutil.WriteFile(filepath.Join(baseDir, "test", "data"), []byte(test.contents), 0666)
f, err := ioutil.TempFile(baseDir, "data")
require.NoError(t, err)
n, err := f.Write([]byte(test.contents))
require.NoError(t, err)
require.Equal(t, len(test.contents), n)
// Run the import statement.
sqlDB.Exec(t, fmt.Sprintf("CREATE TABLE t (%s)", test.create))
sqlDB.Exec(t, fmt.Sprintf("IMPORT INTO t (%s) %s DATA ($1)", test.intoCols, test.typ), "nodelocal://0/test/data")
sqlDB.Exec(t,
fmt.Sprintf("IMPORT INTO t (%s) %s DATA ($1)", test.intoCols, test.typ),
fmt.Sprintf("nodelocal://0/%s", filepath.Base(f.Name())))
// Ensure that the table data is as we expect.
sqlDB.CheckQueryResults(t, test.verifyQuery, test.expected)
// Clean up after the test.
Expand Down Expand Up @@ -1492,7 +1493,7 @@ func TestImportCSVStmt(t *testing.T) {
blockGC := make(chan struct{})

ctx := context.Background()
baseDir := filepath.Join("testdata", "csv")
baseDir := testutils.TestDataPath("testdata", "csv")
tc := testcluster.StartTestCluster(t, nodes, base.TestClusterArgs{ServerArgs: base.TestServerArgs{
SQLMemoryPoolSize: 256 << 20,
ExternalIODir: baseDir,
Expand Down Expand Up @@ -1532,27 +1533,7 @@ func TestImportCSVStmt(t *testing.T) {
}

// Table schema used in IMPORT TABLE tests.
tablePath := filepath.Join(baseDir, "table")
if err := ioutil.WriteFile(tablePath, []byte(`
CREATE TABLE t (
a int8 primary key,
b string,
index (b),
index (a, b)
)
`), 0666); err != nil {
t.Fatal(err)
}
schema := []interface{}{"nodelocal://0/table"}

if err := ioutil.WriteFile(filepath.Join(baseDir, "empty.csv"), nil, 0666); err != nil {
t.Fatal(err)
}

if err := ioutil.WriteFile(filepath.Join(baseDir, "empty.schema"), nil, 0666); err != nil {
t.Fatal(err)
}

empty := []string{"'nodelocal://0/empty.csv'"}
emptySchema := []interface{}{"nodelocal://0/empty.schema"}

Expand Down Expand Up @@ -2375,7 +2356,7 @@ func TestImportIntoCSV(t *testing.T) {
rowsPerRaceFile := 16

ctx := context.Background()
baseDir := filepath.Join("testdata", "csv")
baseDir := testutils.TestDataPath("testdata", "csv")
tc := testcluster.StartTestCluster(t, nodes, base.TestClusterArgs{ServerArgs: base.TestServerArgs{ExternalIODir: baseDir}})
defer tc.Stopper().Stop(ctx)
conn := tc.Conns[0]
Expand Down Expand Up @@ -2418,9 +2399,6 @@ func TestImportIntoCSV(t *testing.T) {
rowsPerFile = rowsPerRaceFile
}

if err := ioutil.WriteFile(filepath.Join(baseDir, "empty.csv"), nil, 0666); err != nil {
t.Fatal(err)
}
empty := []string{"'nodelocal://0/empty.csv'"}

// Support subtests by keeping track of the number of jobs that are executed.
Expand Down
1 change: 1 addition & 0 deletions pkg/testutils/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go_library(
name = "testutils",
srcs = [
"base.go",
"bazel.go",
"dir.go",
"error.go",
"files.go",
Expand Down
120 changes: 120 additions & 0 deletions pkg/testutils/bazel.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// Copyright 2020 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.

package testutils

import (
"os"
"path"
"strings"
"testing"

"github.com/cockroachdb/errors"
)

//
// Utilities in this file are intended to provide helpers for tests
// to access system resources in a correct way, when executing under bazel.
//
// See https://docs.bazel.build/versions/master/test-encyclopedia.html for more details
// on the directory layout.
//
// When test is compiled with bazel, bazel will create a directory, called RUNFILES directory,
// containing all of the resources required to execute such test (libraries, final binary,
// data resources, etc).
//
// Bazel also sets up various environmental variables that point to the location(s) of
// those resources.
//

// Name of the environment variable pointing to the absolute path to the base of the RUNFILES tree.
const testSrcDirEnv = "TEST_SRCDIR"

// Name of the environment variable containing the name of the "workspace".
const testWorkspaceEnv = "TEST_WORKSPACE"

// Name of the environment variable pointing to the absolute path of the
// temporary directory created for the execution of the test.
const testTmpDirEnv = "TEST_TMPDIR"

// Name of the environment variable containing the bazel target path (//pkg/cmd/foo:bar).
const testTargetEnv = "TEST_TARGET"

// RunningUnderBazel returns true if the test is executed by bazel.
func RunningUnderBazel() bool {
return os.Getenv(testSrcDirEnv) != ""
}

func requireEnv(env string) string {
if v := os.Getenv(env); v != "" {
return v
}
panic(errors.AssertionFailedf("expected value for env: %s", env))
}

// TestSrcDir returns the path to the "source" tree.
//
// If running under bazel, this will point to a private, *readonly*
// directory containing symlinks (or copies) of the test data dependencies.
// This directory must be treated readonly. It's an error to try to modify
// anything under this directory: though the operation may succeed, the test
// would not be hermetic, and may fail under other environments.
func TestSrcDir() string {
// If testSrcDirEnv is not set, it means we are not running under bazel,
// and so we can use "" as our directory which should point to the
// src root.
if srcDir := os.Getenv(testSrcDirEnv); srcDir != "" {
return srcDir
}
return ""
}

// TestTempDir returns a temporary directory and a cleanup function for a test.
func TestTempDir(t testing.TB) (string, func()) {
if RunningUnderBazel() {
// Bazel sets up private temp directories for each test.
return requireEnv(testTmpDirEnv), func() {}
}
return TempDir(t)
}

// bazeRelativeTargetPath returns relative path to the package
// of the current test.
func bazelRelativeTargetPath() string {
target := os.Getenv(testTargetEnv)
if target == "" {
return ""
}

// Drop target name.
if last := strings.LastIndex(target, ":"); last > 0 {
target = target[:last]
}
return strings.TrimPrefix(target, "//")
}

// TestDataPath returns a path to the directory containing test data files.
//
// Test files are usually checked into the repository under "testdata" directory.
// If we are not using bazel, then the test executes in the directory of
// the actual test, so the files can be referenced via "testdata/subdir/file" relative path.
//
// However, if we are running under bazel, the data files are specified
// via go_test "data" attribute. These files, in turn, are available under RUNFILES directory.
// This helper attempts to construct appropriate path to the RUNFILES directory
// containing test data files, given the relative (to the test) path components.
//
func TestDataPath(relative ...string) string {
if RunningUnderBazel() {
return path.Join(TestSrcDir(), requireEnv(testWorkspaceEnv), bazelRelativeTargetPath(),
path.Join(relative...))
}
return path.Join(relative...)
}
1 change: 1 addition & 0 deletions pkg/testutils/lint/lint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,7 @@ func TestLint(t *testing.T) {
":!nightly",
":!testutils/lint",
":!util/envutil/env.go",
":!testutils/bazel.go",
":!util/log/tracebacks.go",
":!util/sdnotify/sdnotify_unix.go",
},
Expand Down

0 comments on commit 3c9e7b6

Please sign in to comment.