diff --git a/build/bazelutil/bazel-generate.sh b/build/bazelutil/bazel-generate.sh index 812c95b6bcdb..8953d6a9a5e9 100755 --- a/build/bazelutil/bazel-generate.sh +++ b/build/bazelutil/bazel-generate.sh @@ -53,6 +53,12 @@ fi bazel run //:gazelle +if files_unchanged_from_upstream $(find ./pkg -name '*.proto'); then + echo "Skipping generation of protobuf dependencies." +else + bazel run pkg/gen/genbzl --run_under="cd $PWD && " -- --out-dir pkg/gen +fi + if files_unchanged_from_upstream $(find ./pkg -name BUILD.bazel) $(find ./pkg -name '*.bzl'); then echo "Skipping //pkg/cmd/generate-test-suites (relevant files are unchanged from upstream)." else diff --git a/pkg/gen/BUILD.bazel b/pkg/gen/BUILD.bazel index 458c83fc753f..81df468684d0 100644 --- a/pkg/gen/BUILD.bazel +++ b/pkg/gen/BUILD.bazel @@ -6,70 +6,6 @@ hoist_files( tags = ["no-remote-exec"], ) -# bazel query 'kind(go_proto_library, //pkg/...)' | sort | awk '{ printf("\"%s\",\n", $0) }' - go_proto_srcs( name = "go_proto_srcs", - srcs = [ - "//pkg/acceptance/cluster:cluster_go_proto", - "//pkg/blobs/blobspb:blobspb_go_proto", - "//pkg/build:build_go_proto", - "//pkg/ccl/backupccl:backupccl_go_proto", - "//pkg/ccl/baseccl:baseccl_go_proto", - "//pkg/ccl/sqlproxyccl/tenant:tenant_go_proto", - "//pkg/ccl/storageccl/engineccl/enginepbccl:enginepbccl_go_proto", - "//pkg/ccl/streamingccl/streampb:streampb_go_proto", - "//pkg/ccl/utilccl/licenseccl:licenseccl_go_proto", - "//pkg/clusterversion:clusterversion_go_proto", - "//pkg/config:config_go_proto", - "//pkg/config/zonepb:zonepb_go_proto", - "//pkg/geo/geoindex:geoindex_go_proto", - "//pkg/geo/geopb:geopb_go_proto", - "//pkg/gossip:gossip_go_proto", - "//pkg/jobs/jobspb:jobspb_go_proto", - "//pkg/kv/kvnemesis:kvnemesis_go_proto", - "//pkg/kv/kvserver:kvserver_go_proto", - "//pkg/kv/kvserver/closedts/ctpb:ctpb_go_proto", - "//pkg/kv/kvserver/concurrency/lock:lock_go_proto", - "//pkg/kv/kvserver/kvserverpb:kvserverpb_go_proto", - "//pkg/kv/kvserver/liveness/livenesspb:livenesspb_go_proto", - "//pkg/kv/kvserver/loqrecovery/loqrecoverypb:loqrecoverypb_go_proto", - "//pkg/kv/kvserver/protectedts/ptpb:ptpb_go_proto", - "//pkg/kv/kvserver/protectedts/ptstorage:ptstorage_go_proto", - "//pkg/kv/kvserver/readsummary/rspb:rspb_go_proto", - "//pkg/roachpb:roachpb_go_proto", - "//pkg/rpc:rpc_go_proto", - "//pkg/server/diagnostics/diagnosticspb:diagnosticspb_go_proto", - "//pkg/server/serverpb:serverpb_go_proto", - "//pkg/server/status/statuspb:statuspb_go_proto", - "//pkg/sql/catalog/catpb:catpb_go_proto", - "//pkg/sql/catalog/descpb:descpb_go_proto", - "//pkg/sql/contentionpb:contentionpb_go_proto", - "//pkg/sql/execinfrapb:execinfrapb_go_proto", - "//pkg/sql/inverted:inverted_go_proto", - "//pkg/sql/pgwire/pgerror:pgerror_go_proto", - "//pkg/sql/protoreflect/test:protoreflecttest_go_proto", - "//pkg/sql/rowenc/rowencpb:rowencpb_go_proto", - "//pkg/sql/schemachanger/scpb:scpb_go_proto", - "//pkg/sql/sessiondatapb:sessiondatapb_go_proto", - "//pkg/sql/sqlstats/persistedsqlstats:persistedsqlstats_go_proto", - "//pkg/sql/stats:stats_go_proto", - "//pkg/sql/types:types_go_proto", - "//pkg/startupmigrations/leasemanager:leasemanager_go_proto", - "//pkg/storage/enginepb:enginepb_go_proto", - "//pkg/testutils/grpcutils:grpcutils_go_proto", - "//pkg/ts/catalog:catalog_go_proto", - "//pkg/ts/tspb:tspb_go_proto", - "//pkg/util:util_go_proto", - "//pkg/util/duration:duration_go_proto", - "//pkg/util/hlc:hlc_go_proto", - "//pkg/util/log/eventpb:eventpb_go_proto", - "//pkg/util/log/logpb:logpb_go_proto", - "//pkg/util/metric:metric_go_proto", - "//pkg/util/optional:optional_go_proto", - "//pkg/util/protoutil:protoutil_go_proto", - "//pkg/util/timeutil/pgdate:pgdate_go_proto", - "//pkg/util/tracing/tracingpb:tracingpb_go_proto", - "//pkg/util/tracing/tracingservicepb:tracingservicepb_go_proto", - ], ) diff --git a/pkg/gen/gen.bzl b/pkg/gen/gen.bzl index b0ff0e4d7fc7..d24643705ed6 100644 --- a/pkg/gen/gen.bzl +++ b/pkg/gen/gen.bzl @@ -1,29 +1,48 @@ load("@io_bazel_rules_go//go:def.bzl", "GoSource") +load(":protobuf.bzl", "PROTOBUF_SRCS") GeneratedFileInfo = provider( "Info needed to hoist generated files", fields = { "generated_files": "dictionary from prefix to list of files", + "cleanup_tasks": "list of bash commands to run" } ) def _go_proto_srcs_impl(ctx): generated_files = {} - for s in ctx.attr.srcs: + for s in ctx.attr._srcs: srcs = s[GoSource] lbl = srcs.library.label imp = srcs.library.importpath imp = imp[:imp.find(lbl.package)] prefix = "{}/{}_/{}".format(lbl.package, lbl.name, imp) generated_files[prefix] = [f for f in srcs.srcs] + return [ - GeneratedFileInfo(generated_files = generated_files) + GeneratedFileInfo( + generated_files = generated_files, + # Create a task to remove any existing protobuf files. + cleanup_tasks = [ +# Use a subshell with () to avoid changing the directory in the main shell. +""" +( + cd "${BUILD_WORKSPACE_DIRECTORY}" + # Avoid searching the node_modules directory because it's full of + # irrelevant files. + find ./pkg -name node_modules -prune -o \ + -type f -name '*.pb.go' -exec rm {} + -o \ + -type f -name '*.pb.gw.go' -exec rm {} + +) +""", + ], + ) ] go_proto_srcs = rule( implementation = _go_proto_srcs_impl, attrs = { - "srcs": attr.label_list(providers = [GoSource]), + "_srcs": attr.label_list(providers = [GoSource], default=PROTOBUF_SRCS), }, ) @@ -35,14 +54,18 @@ chmod +w {dst} _script_fmt = """#!/bin/bash set -euo pipefail -{} +{cleanup_tasks} +{cmds} """ def _hoist_files_impl(ctx): + cleanup_tasks = [] cmds = [] generated_files = {} for set in ctx.attr.data: - for prefix, files in set[GeneratedFileInfo].generated_files.items(): + gfi = set[GeneratedFileInfo] + cleanup_tasks += gfi.cleanup_tasks + for prefix, files in gfi.generated_files.items(): if prefix not in generated_files: generated_files[prefix] = [] for file in files: @@ -54,13 +77,19 @@ def _hoist_files_impl(ctx): generated_files[prefix].append(file) executable = ctx.actions.declare_file(ctx.label.name) - script = _script_fmt.format("\n".join(cmds)) + script = _script_fmt.format( + cleanup_tasks = "\n".join(cleanup_tasks), + cmds = "\n".join(cmds), + ) ctx.actions.write(executable, script, is_executable=True) runfiles = ctx.runfiles(files = [file for files in generated_files.values() for file in files]) return [ DefaultInfo(executable = executable, runfiles=runfiles), - GeneratedFileInfo(generated_files = generated_files) + GeneratedFileInfo( + generated_files = generated_files, + cleanup_tasks = cleanup_tasks, + ) ] hoist_files = rule( diff --git a/pkg/gen/genbzl/BUILD.bazel b/pkg/gen/genbzl/BUILD.bazel new file mode 100644 index 000000000000..9044b3d149a4 --- /dev/null +++ b/pkg/gen/genbzl/BUILD.bazel @@ -0,0 +1,18 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "genbzl_lib", + srcs = ["main.go"], + importpath = "github.com/cockroachdb/cockroach/pkg/gen/genbzl", + visibility = ["//visibility:private"], + deps = [ + "//pkg/cli/exit", + "@com_github_cockroachdb_errors//:errors", + ], +) + +go_binary( + name = "genbzl", + embed = [":genbzl_lib"], + visibility = ["//visibility:public"], +) diff --git a/pkg/gen/genbzl/main.go b/pkg/gen/genbzl/main.go new file mode 100644 index 000000000000..eebad827400e --- /dev/null +++ b/pkg/gen/genbzl/main.go @@ -0,0 +1,140 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +// Command genbzl is used to generate bazel files which then get imported by +// the gen package's BUILD.bazel to facilitate hoisting these generated files +// back into the source tree. +// +// It's all a bit meta. The flow is that we invoke this binary inside the +// bazelutil/bazel-generate.sh script which writes out some bzl files with +// lists of targets which are then depended on in gen.bzl +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "text/template" + + "github.com/cockroachdb/cockroach/pkg/cli/exit" + "github.com/cockroachdb/errors" +) + +var ( + outDir = flag.String("out-dir", "", "directory in which to place the generated files") +) + +func main() { + flag.Parse() + if err := generate(*outDir); err != nil { + fmt.Fprintf(os.Stderr, "failed to generate files: %v\n", err) + exit.WithCode(exit.UnspecifiedError()) + } +} + +var targets = []*target{ + newTarget( + "protobuf.bzl", + newQuery("kind(go_proto_library, //pkg/...)"), + `# Generated by genbzl + +PROTOBUF_SRCS = [{{ range . }} + "{{ . }}",{{end}} +] +`), +} + +func newQuery(q string, filters ...func(s string) (shouldKeep bool)) *query { + return &query{ + query: q, + filters: filters, + } +} + +type target struct { + filename string + template *template.Template + query *query +} + +func newTarget(filename string, query *query, tmpl string) *target { + return &target{ + filename: filename, + template: template.Must(template.New(filename).Parse(tmpl)), + query: query, + } +} + +type query struct { + query string + filters []func(s string) bool +} + +func (q query) exec() (results []string, _ error) { + cmd := exec.Command("bazel", "query", q.query) + var stdout bytes.Buffer + var stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, + "failed to run %s: (stderr)\n%s", cmd, &stderr) + } + sc := bufio.NewScanner(&stdout) + keep := func(s string) (shouldKeep bool) { + for _, f := range q.filters { + if !f(s) { + return false + } + } + return true + } + for sc.Scan() { + if s := sc.Text(); keep(s) { + results = append(results, s) + } + } + sort.Strings(results) + return results, nil +} + +func generate(outDir string) error { + for _, t := range targets { + out, err := t.query.exec() + if err != nil { + return err + } + if err != nil { + return err + } + var buf bytes.Buffer + if err := t.template.Execute(&buf, out); err != nil { + return errors.Wrapf(err, "failed to execute template for %s", t.filename) + } + f, err := os.Create(filepath.Join(outDir, t.filename)) + if err != nil { + return errors.Wrapf(err, "failed to open file for %s", t.filename) + } + if _, err := io.Copy(f, &buf); err != nil { + return errors.Wrapf(err, "failed to write file for %s", t.filename) + } + if err := f.Close(); err != nil { + return errors.Wrapf(err, "failed to write file for %s", t.filename) + } + } + return nil +} diff --git a/pkg/gen/protobuf.bzl b/pkg/gen/protobuf.bzl new file mode 100644 index 000000000000..f82ead8e7e48 --- /dev/null +++ b/pkg/gen/protobuf.bzl @@ -0,0 +1,64 @@ +# Generated by genbzl + +PROTOBUF_SRCS = [ + "//pkg/acceptance/cluster:cluster_go_proto", + "//pkg/blobs/blobspb:blobspb_go_proto", + "//pkg/build:build_go_proto", + "//pkg/ccl/backupccl:backupccl_go_proto", + "//pkg/ccl/baseccl:baseccl_go_proto", + "//pkg/ccl/sqlproxyccl/tenant:tenant_go_proto", + "//pkg/ccl/storageccl/engineccl/enginepbccl:enginepbccl_go_proto", + "//pkg/ccl/streamingccl/streampb:streampb_go_proto", + "//pkg/ccl/utilccl/licenseccl:licenseccl_go_proto", + "//pkg/clusterversion:clusterversion_go_proto", + "//pkg/config/zonepb:zonepb_go_proto", + "//pkg/config:config_go_proto", + "//pkg/geo/geoindex:geoindex_go_proto", + "//pkg/geo/geopb:geopb_go_proto", + "//pkg/gossip:gossip_go_proto", + "//pkg/jobs/jobspb:jobspb_go_proto", + "//pkg/kv/kvnemesis:kvnemesis_go_proto", + "//pkg/kv/kvserver/closedts/ctpb:ctpb_go_proto", + "//pkg/kv/kvserver/concurrency/lock:lock_go_proto", + "//pkg/kv/kvserver/kvserverpb:kvserverpb_go_proto", + "//pkg/kv/kvserver/liveness/livenesspb:livenesspb_go_proto", + "//pkg/kv/kvserver/loqrecovery/loqrecoverypb:loqrecoverypb_go_proto", + "//pkg/kv/kvserver/protectedts/ptpb:ptpb_go_proto", + "//pkg/kv/kvserver/protectedts/ptstorage:ptstorage_go_proto", + "//pkg/kv/kvserver/readsummary/rspb:rspb_go_proto", + "//pkg/kv/kvserver:kvserver_go_proto", + "//pkg/roachpb:roachpb_go_proto", + "//pkg/rpc:rpc_go_proto", + "//pkg/server/diagnostics/diagnosticspb:diagnosticspb_go_proto", + "//pkg/server/serverpb:serverpb_go_proto", + "//pkg/server/status/statuspb:statuspb_go_proto", + "//pkg/sql/catalog/catpb:catpb_go_proto", + "//pkg/sql/catalog/descpb:descpb_go_proto", + "//pkg/sql/contentionpb:contentionpb_go_proto", + "//pkg/sql/execinfrapb:execinfrapb_go_proto", + "//pkg/sql/inverted:inverted_go_proto", + "//pkg/sql/pgwire/pgerror:pgerror_go_proto", + "//pkg/sql/protoreflect/test:protoreflecttest_go_proto", + "//pkg/sql/rowenc/rowencpb:rowencpb_go_proto", + "//pkg/sql/schemachanger/scpb:scpb_go_proto", + "//pkg/sql/sessiondatapb:sessiondatapb_go_proto", + "//pkg/sql/sqlstats/persistedsqlstats:persistedsqlstats_go_proto", + "//pkg/sql/stats:stats_go_proto", + "//pkg/sql/types:types_go_proto", + "//pkg/startupmigrations/leasemanager:leasemanager_go_proto", + "//pkg/storage/enginepb:enginepb_go_proto", + "//pkg/testutils/grpcutils:grpcutils_go_proto", + "//pkg/ts/catalog:catalog_go_proto", + "//pkg/ts/tspb:tspb_go_proto", + "//pkg/util/duration:duration_go_proto", + "//pkg/util/hlc:hlc_go_proto", + "//pkg/util/log/eventpb:eventpb_go_proto", + "//pkg/util/log/logpb:logpb_go_proto", + "//pkg/util/metric:metric_go_proto", + "//pkg/util/optional:optional_go_proto", + "//pkg/util/protoutil:protoutil_go_proto", + "//pkg/util/timeutil/pgdate:pgdate_go_proto", + "//pkg/util/tracing/tracingpb:tracingpb_go_proto", + "//pkg/util/tracing/tracingservicepb:tracingservicepb_go_proto", + "//pkg/util:util_go_proto", +]