Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
79245: ci: build and publish cockroach-sql r=rickystewart a=rail

This patch adds `cockroach-sql` as a separate artifact published to S3.
The binary is not stripped (TBD).

Fixes cockroachdb#75800

Release note: None

79651: test: fix bazel sqllite invocation r=cucaroach a=cucaroach

Release note: None

79661: WORKSPACE: update rules_go version r=mari-crl a=mari-crl

The rules_go version has been bumped to accept the new changes from
cockroachdb/rules_go#5.

Finishing blow for cockroachdb#76377.

Release note: None

Co-authored-by: Rail Aliiev <[email protected]>
Co-authored-by: Tommy Reilly <[email protected]>
Co-authored-by: Mari Staib <[email protected]>
  • Loading branch information
4 people committed Apr 8, 2022
4 parents 6434277 + 590fcf1 + 8edf457 + a495359 commit fb54ea7
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 35 deletions.
8 changes: 4 additions & 4 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
# Load go bazel tools. This gives us access to the go bazel SDK/toolchains.
http_archive(
name = "io_bazel_rules_go",
sha256 = "a80c384a85ab9f6318419ed38a39b13e8481e9d2ea25a696ea9e25438e017b5f",
strip_prefix = "cockroachdb-rules_go-f96cc04",
sha256 = "8a993815a6a7cfe47e46238383f0e36a80b9ce0ac482f855c7288a31565b5661",
strip_prefix = "cockroachdb-rules_go-58cb947",
urls = [
# cockroachdb/rules_go as of f96cc0445102a7b08bceeb0bba32b4d82467a065
# cockroachdb/rules_go as of 58cb94707783f529462d4c6d3c698933a1022942
# (upstream release-0.29 plus a few patches).
"https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-54-gf96cc04.tar.gz",
"https://storage.googleapis.com/public-bazel-artifacts/bazel/cockroachdb-rules_go-v0.27.0-56-g58cb947.tar.gz",
],
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ GO_TEST_JSON_OUTPUT_FILE=/artifacts/test.json.txt
exit_status=0
$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci --config=crdb_test_off \
test //pkg/sql/logictest:logictest_test -- \
--test_arg -bigtest --test_arg -flex-types --test_arg -parallel=4 \
--test_arg -bigtest --test_arg -flex-types --test_arg -test.parallel=4 \
--test_timeout 86400 --test_filter '^TestSqlLiteLogic$|^TestTenantSQLLiteLogic$' \
--test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE || exit_status=$?
process_test_json \
Expand Down
19 changes: 12 additions & 7 deletions pkg/cmd/publish-provisional-artifacts/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ func run(svc s3I, flags runFlags, execFn release.ExecFn) {
o.VersionStr = versionStr
o.BucketName = bucketName
o.AbsolutePath = filepath.Join(flags.pkgDir, "cockroach"+release.SuffixFromPlatform(platform))

o.CockroachSQLAbsolutePath = filepath.Join(flags.pkgDir, "cockroach-sql"+release.SuffixFromPlatform(platform))
cockroachBuildOpts = append(cockroachBuildOpts, o)
}

Expand Down Expand Up @@ -224,9 +224,10 @@ type opts struct {

Platform release.Platform

AbsolutePath string
BucketName string
PkgDir string
AbsolutePath string
CockroachSQLAbsolutePath string
BucketName string
PkgDir string
}

func putNonRelease(svc s3I, o opts) {
Expand All @@ -236,15 +237,18 @@ func putNonRelease(svc s3I, o opts) {
Branch: o.Branch,
BucketName: o.BucketName,
Files: append(
[]release.NonReleaseFile{release.MakeCRDBBinaryNonReleaseFile(o.AbsolutePath, o.VersionStr)},
[]release.NonReleaseFile{
release.MakeCRDBBinaryNonReleaseFile(o.AbsolutePath, o.VersionStr),
release.MakeCRDBBinaryNonReleaseFile(o.CockroachSQLAbsolutePath, o.VersionStr),
},
release.MakeCRDBLibraryNonReleaseFiles(o.PkgDir, o.Platform, o.VersionStr)...,
),
},
)
}

func s3KeyRelease(o opts) (string, string) {
return release.S3KeyRelease(o.Platform, o.VersionStr)
return release.S3KeyRelease(o.Platform, o.VersionStr, "cockroach")
}

func putRelease(svc s3I, o opts) {
Expand All @@ -254,9 +258,10 @@ func putRelease(svc s3I, o opts) {
Platform: o.Platform,
VersionStr: o.VersionStr,
Files: append(
[]release.ArchiveFile{release.MakeCRDBBinaryArchiveFile(o.AbsolutePath)},
[]release.ArchiveFile{release.MakeCRDBBinaryArchiveFile(o.AbsolutePath, "cockroach")},
release.MakeCRDBLibraryArchiveFiles(o.PkgDir, o.Platform)...,
),
ExtraFiles: []release.ArchiveFile{release.MakeCRDBBinaryArchiveFile(o.CockroachSQLAbsolutePath, "cockroach-sql")},
})
}

Expand Down
72 changes: 56 additions & 16 deletions pkg/cmd/publish-provisional-artifacts/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ func (r *mockExecRunner) run(c *exec.Cmd) ([]byte, error) {
}
if c.Args[0] == "bazel" && c.Args[1] == "build" {
path := filepath.Join(r.fakeBazelBin, "pkg", "cmd", "cockroach", "cockroach_", "cockroach")
pathSQL := filepath.Join(r.fakeBazelBin, "pkg", "cmd", "cockroach-sql", "cockroach-sql_", "cockroach-sql")
var platform release.Platform
for _, arg := range c.Args {
if strings.HasPrefix(arg, `--config=`) {
Expand All @@ -105,13 +106,14 @@ func (r *mockExecRunner) run(c *exec.Cmd) ([]byte, error) {
case "crosswindowsbase":
platform = release.PlatformWindows
path += ".exe"
pathSQL += ".exe"
case "ci", "with_ui":
default:
panic(fmt.Sprintf("Unexpected configuration %s", arg))
}
}
}
paths = append(paths, path)
paths = append(paths, path, pathSQL)
ext := release.SharedLibraryExtensionFromPlatform(platform)
for _, lib := range release.CRDBSharedLibraries {
libDir := "lib"
Expand Down Expand Up @@ -151,27 +153,36 @@ func TestProvisional(t *testing.T) {
branch: `provisional_201901010101_v0.0.1-alpha`,
},
expectedCmds: []string{
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"env=[] args=bazel info bazel-bin -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"env=[MALLOC_CONF=prof:true] args=./cockroach.linux-2.6.32-gnu-amd64 version",
"env=[] args=ldd ./cockroach.linux-2.6.32-gnu-amd64",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"env=[] args=bazel info bazel-bin -c opt --config=ci --config=with_ui --config=crossmacosbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=." +
"/build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"env=[] args=bazel info bazel-bin -c opt --config=ci --config=with_ui --config=crosswindowsbase",
},
expectedGets: nil,
expectedPuts: []string{
"s3://binaries.cockroachdb.com/cockroach-v0.0.1-alpha.linux-amd64.tgz " +
"CONTENTS <binary stuff>",
"s3://binaries.cockroachdb.com/cockroach-v0.0.1-alpha.linux-amd64.tgz.sha256sum CONTENTS <sha256sum>",
"s3://binaries.cockroachdb.com/cockroach-sql-v0.0.1-alpha.linux-amd64 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"s3://binaries.cockroachdb.com/cockroach-v0.0.1-alpha.darwin-10.9-amd64.tgz " +
"CONTENTS <binary stuff>",
"s3://binaries.cockroachdb.com/cockroach-v0.0.1-alpha.darwin-10.9-amd64.tgz.sha256sum CONTENTS <sha256sum>",
"s3://binaries.cockroachdb.com/cockroach-sql-v0.0.1-alpha.darwin-10.9-amd64 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"s3://binaries.cockroachdb.com/cockroach-v0.0.1-alpha.windows-6.2-amd64.zip " +
"CONTENTS <binary stuff>",
"s3://binaries.cockroachdb.com/cockroach-v0.0.1-alpha.windows-6.2-amd64.zip." +
"sha256sum CONTENTS <sha256sum>",
"s3://binaries.cockroachdb.com/cockroach-sql-v0.0.1-alpha.windows-6.2-amd64.exe " +
"CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql" +
" '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary v0.0.1-alpha release' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
},
},
{
Expand All @@ -183,40 +194,69 @@ func TestProvisional(t *testing.T) {
sha: `00SHA00`,
},
expectedCmds: []string{
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=." +
"/build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"env=[] args=bazel info bazel-bin -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"env=[MALLOC_CONF=prof:true] args=./cockroach.linux-2.6.32-gnu-amd64 version",
"env=[] args=ldd ./cockroach.linux-2.6.32-gnu-amd64",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"env=[] args=bazel info bazel-bin -c opt --config=ci --config=with_ui --config=crossmacosbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"env=[] args=bazel info bazel-bin -c opt --config=ci --config=with_ui --config=crosswindowsbase",
},
expectedGets: nil,
expectedPuts: []string{
"s3://cockroach//cockroach/cockroach.linux-gnu-amd64.00SHA00 " +
"CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp." +
"sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/cockroach.linux-gnu-amd64.LATEST/no-cache " +
"REDIRECT /cockroach/cockroach.linux-gnu-amd64.00SHA00",
"s3://cockroach//cockroach/lib/libgeos.linux-gnu-amd64.00SHA00.so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"s3://cockroach//cockroach/cockroach-sql.linux-gnu-amd64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/cockroach-sql.linux-gnu-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.linux-gnu-amd64.00SHA00",
"s3://cockroach//cockroach/lib/libgeos.linux-gnu-amd64.00SHA00." +
"so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/lib/libgeos.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.linux-gnu-amd64.00SHA00.so",
"s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-amd64.00SHA00.so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"s3://cockroach//cockroach/lib/libgeos_c.linux-gnu-amd64.00SHA00." +
"so CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary' -c opt --config=ci --config=with_ui --config=crosslinuxbase",
"s3://cockroach/cockroach/lib/libgeos_c.linux-gnu-amd64.so.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.linux-gnu-amd64.00SHA00.so",
"s3://cockroach//cockroach/cockroach.darwin-amd64.00SHA00 " +
"CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/cockroach.darwin-amd64.LATEST/no-cache " +
"REDIRECT /cockroach/cockroach.darwin-amd64.00SHA00",
"s3://cockroach//cockroach/lib/libgeos.darwin-amd64.00SHA00.dylib CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"s3://cockroach//cockroach/cockroach-sql.darwin-amd64.00SHA00 CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/cockroach-sql.darwin-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql." +
"darwin-amd64.00SHA00",
"s3://cockroach//cockroach/lib/libgeos.darwin-amd64.00SHA00." +
"dylib CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/lib/libgeos.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.darwin-amd64.00SHA00.dylib",
"s3://cockroach//cockroach/lib/libgeos_c.darwin-amd64.00SHA00.dylib CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"s3://cockroach//cockroach/lib/libgeos_c.darwin-amd64.00SHA00." +
"dylib CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp." +
"sh x86_64-apple-darwin19 official-binary' -c opt --config=ci --config=with_ui --config=crossmacosbase",
"s3://cockroach/cockroach/lib/libgeos_c.darwin-amd64.dylib.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.darwin-amd64.00SHA00.dylib",
"s3://cockroach//cockroach/cockroach.windows-amd64.00SHA00.exe " +
"CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp." +
"sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/cockroach.windows-amd64.LATEST/no-cache " +
"REDIRECT /cockroach/cockroach.windows-amd64.00SHA00.exe",
"s3://cockroach//cockroach/lib/libgeos.windows-amd64.00SHA00.dll CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"s3://cockroach//cockroach/cockroach-sql.windows-amd64.00SHA00.exe CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/cockroach-sql.windows-amd64.LATEST/no-cache REDIRECT /cockroach/cockroach-sql.windows-amd64.00SHA00.exe",
"s3://cockroach//cockroach/lib/libgeos.windows-amd64.00SHA00." +
"dll CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/lib/libgeos.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos.windows-amd64.00SHA00.dll",
"s3://cockroach//cockroach/lib/libgeos_c.windows-amd64.00SHA00.dll CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"s3://cockroach//cockroach/lib/libgeos_c.windows-amd64.00SHA00." +
"dll CONTENTS env=[] args=bazel build //pkg/cmd/cockroach //c-deps:libgeos //pkg/cmd/cockroach-sql " +
"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary' -c opt --config=ci --config=with_ui --config=crosswindowsbase",
"s3://cockroach/cockroach/lib/libgeos_c.windows-amd64.dll.LATEST/no-cache REDIRECT /cockroach/lib/libgeos_c.windows-amd64.00SHA00.dll",
},
},
Expand Down
Loading

0 comments on commit fb54ea7

Please sign in to comment.