bazel: build http docs w/ bazel

We generate this in a different way than the `Makefile` does -- we take
all the proto descriptor sets (that we need to generate anyway as part
of the build) and pass them in to `protoc` rather than passing a bunch
of include paths, which wouldn't work terribly well in the sandbox.

Also teach CI and `dev` to start building these docs.

I deleted a stray comment in `pkg/server/serverpb/admin.proto` that was
causing a diff in the generated file. I also had to patch
`@com_github_pseudomuto_protoc_gen_doc` to quash a compiler error (I
just deleted a dependency that isn't actually necessary).

Closes cockroachdb#65814.

Release note: None

DEPS.bzl

@@ -1082,6 +1082,7 @@ def go_deps():
     go_repository(
         name = "com_github_envoyproxy_protoc_gen_validate",
         build_file_proto_mode = "disable_global",
+        build_naming_convention = "go_default_library",
         importpath = "github.com/envoyproxy/protoc-gen-validate",
         sum = "h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=",
         version = "v0.1.0",
@@ -3478,6 +3479,10 @@ def go_deps():
         name = "com_github_pseudomuto_protoc_gen_doc",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pseudomuto/protoc-gen-doc",
+        patch_args = ["-p1"],
+        patches = [
+            "@cockroach//build/patches:com_github_pseudomuto_protoc_gen_doc.patch",
+        ],
         sum = "h1:61vWZuxYa8D7Rn4h+2dgoTNqnluBmJya2MgbqO32z6g=",
         version = "v1.3.2",
     )

Makefile

@@ -1556,7 +1556,7 @@ bin/.docgen_http: bin/docgen bin/.bootstrap
 	docgen http \
 	--gendoc ./bin/protoc-gen-doc \
 	--out docs/generated/http \
-	--protobuf "-Ipkg -I$(GOGO_PROTOBUF_PATH) -I$(COREOS_PATH) -I$(GRPC_GATEWAY_GOOGLEAPIS_PATH) -I$(ERRORS_PATH) -I$(PROMETHEUS_PATH)"
+	--protoc-flags "-Ipkg -I$(GOGO_PROTOBUF_PATH) -I$(COREOS_PATH) -I$(GRPC_GATEWAY_GOOGLEAPIS_PATH) -I$(ERRORS_PATH) -I$(PROMETHEUS_PATH) ./pkg/server/serverpb/status.proto ./pkg/server/serverpb/admin.proto ./pkg/server/status/statuspb/status.proto"
 	touch $@
 
 .PHONY: docs/generated/redact_safe.md

build/patches/com_github_pseudomuto_protoc_gen_doc.patch

@@ -0,0 +1,22 @@
+diff -urN a/cmd/protoc-gen-doc/BUILD.bazel b/cmd/protoc-gen-doc/BUILD.bazel
+--- a/cmd/protoc-gen-doc/BUILD.bazel	1969-12-31 19:00:00.000000000 -0500
++++ b/cmd/protoc-gen-doc/BUILD.bazel	2000-01-01 00:00:00.000000000 -0000
+@@ -11,7 +11,6 @@
+     deps = [
+         "//:protoc-gen-doc",
+         "//extensions/google_api_http",
+-        "//extensions/lyft_validate",
+         "//extensions/validator_field",
+         "@com_github_pseudomuto_protokit//:protokit",
+     ],
+diff -urN a/cmd/protoc-gen-doc/main.go b/cmd/protoc-gen-doc/main.go
+--- a/cmd/protoc-gen-doc/main.go	1969-12-31 19:00:00.000000000 -0500
++++ b/cmd/protoc-gen-doc/main.go	2000-01-01 00:00:00.000000000 -0000
+@@ -21,7 +21,6 @@
+
+ 	gendoc "github.com/pseudomuto/protoc-gen-doc"
+ 	_ "github.com/pseudomuto/protoc-gen-doc/extensions/google_api_http" // imported for side effects
+-	_ "github.com/pseudomuto/protoc-gen-doc/extensions/lyft_validate"   // imported for side effects
+ 	_ "github.com/pseudomuto/protoc-gen-doc/extensions/validator_field" // imported for side effects
+ )
+

build/teamcity/cockroach/ci/builds/build_linux_x86_64.sh

@@ -32,6 +32,14 @@ done
 # Ensure the generated docs are inclusive of what we have in tree: list all
 # generated files in a few subdirectories and make sure they're all in the
 # build output.
+for FILE in $(ls $root/docs/generated/http/*.md | xargs -n1 basename)
+do
+    if [[ ! -f $root/artifacts/bazel-bin/docs/generated/http/$FILE ]]
+    then
+        echo "File $root/artifacts/bazel-bin/docs/generated/http/$FILE does not exist as a generated artifact. Is docs/generated/http/BUILD.bazel up-to-date?"
+        FAILED=1
+    fi
+done
 for FILE in $(ls $root/docs/generated/sql/*.md | xargs -n1 basename)
 do
     if [[ ! -f $root/artifacts/bazel-bin/docs/generated/sql/$FILE ]]

docs/generated/bazel_targets.txt

@@ -5,6 +5,7 @@ documentation. Lines not beginning with // should be ignored.
 //docs/generated:gen-logsinks-md
 //docs/generated:gen-eventlog-md
 //docs/generated:gen-logformats-md
+//docs/generated/http
 //docs/generated/settings
 //docs/generated/settings:settings_for_tenants
 //docs/generated/sql

docs/generated/http/BUILD.bazel

@@ -0,0 +1,72 @@
+genrule(
+    name = "http",
+    srcs = [
+        "//pkg/ts/tspb:tspb_proto",
+        "//pkg/util/metric:metric_proto",
+        "//pkg/ts/catalog:catalog_proto",
+        "//pkg/util/duration:duration_proto",
+        "//pkg/util/timeutil/pgdate:pgdate_proto",
+        "//pkg/sql/sessiondatapb:sessiondatapb_proto",
+        "//pkg/sql/inverted:inverted_proto",
+        "//pkg/sql/pgwire/pgerror:pgerror_proto",
+        "//pkg/sql/contentionpb:contentionpb_proto",
+        "//pkg/sql/execinfrapb:execinfrapb_proto",
+        "//pkg/kv/kvserver/kvserverpb:kvserverpb_proto",
+        "//pkg/kv/kvserver/liveness/livenesspb:livenesspb_proto",
+        "//pkg/util/log/logpb:logpb_proto",
+        "//pkg/build:build_proto",
+        "//pkg/clusterversion:clusterversion_proto",
+        "//pkg/config/zonepb:zonepb_proto",
+        "//pkg/geo/geoindex:geoindex_proto",
+        "//pkg/geo/geopb:geopb_proto",
+        "//pkg/gossip:gossip_proto",
+        "//pkg/jobs/jobspb:jobspb_proto",
+        "//pkg/kv/kvserver/concurrency/lock:lock_proto",
+        "//pkg/kv/kvserver/readsummary/rspb:rspb_proto",
+        "//pkg/roachpb:roachpb_proto",
+        "//pkg/server/diagnostics/diagnosticspb:diagnosticspb_proto",
+        "//pkg/server/serverpb:serverpb_proto",
+        "//pkg/server/status/statuspb:statuspb_proto",
+        "//pkg/sql/catalog/descpb:descpb_proto",
+        "//pkg/sql/schemachanger/scpb:scpb_proto",
+        "//pkg/sql/types:types_proto",
+        "//pkg/storage/enginepb:enginepb_proto",
+        "//pkg/util:util_proto",
+        "//pkg/util/hlc:hlc_proto",
+        "//pkg/util/tracing/tracingpb:tracingpb_proto",
+        "@com_github_prometheus_client_model//io/prometheus/client:client_proto",
+        "@com_github_cockroachdb_errors//errorspb:errorspb_proto",
+        "@com_github_gogo_protobuf//gogoproto:gogo_proto",
+        "@com_google_protobuf//:any_proto",
+        "@com_google_protobuf//:descriptor_proto",
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:timestamp_proto",
+        "@go_googleapis//google/api:annotations_proto",
+        "@io_etcd_go_etcd_raft_v3//raftpb:raftpb_proto",
+    ],
+    outs = [
+        "full.md",
+        "health-other.md",
+        "health-request.md",
+        "health-response.md",
+        "hotranges-other.md",
+        "hotranges-request.md",
+        "hotranges-response.md",
+        "nodes-other.md",
+        "nodes-request.md",
+        "nodes-response.md",
+    ],
+    cmd = """
+      DESCRIPTOR_SET_IN=$$(echo $(SRCS) | tr -s '[:blank:]' ':')
+      $(location //pkg/cmd/docgen) \
+          http --protoc $(location @com_google_protobuf//:protoc) \
+          --gendoc $(location @com_github_pseudomuto_protoc_gen_doc//cmd/protoc-gen-doc) \
+          --out $(RULEDIR) \
+          --protoc-flags "--descriptor_set_in $$DESCRIPTOR_SET_IN server/serverpb/status.proto server/serverpb/admin.proto server/status/statuspb/status.proto"
+    """,
+    exec_tools = [
+        "@com_google_protobuf//:protoc",
+        "@com_github_pseudomuto_protoc_gen_doc//cmd/protoc-gen-doc",
+        "//pkg/cmd/docgen",
+    ],
+)

pkg/cmd/docgen/http.go

@@ -26,22 +26,27 @@ import (
 
 func init() {
 	var (
-		protobufPath string
-		genDocPath   string
-		outPath      string
+		protocPath  string
+		protocFlags string
+		genDocPath  string
+		outPath     string
 	)
 
 	cmdHTTP := &cobra.Command{
 		Use:   "http",
 		Short: "Generate HTTP docs",
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := runHTTP(genDocPath, protobufPath, outPath); err != nil {
+			if err := runHTTP(protocPath, genDocPath, protocFlags, outPath); err != nil {
 				fmt.Fprintln(os.Stdout, err)
 				os.Exit(1)
 			}
 		},
 	}
-	cmdHTTP.Flags().StringVar(&protobufPath, "protobuf", "", "Protobuf include paths.")
+	cmdHTTP.Flags().StringVar(&protocPath, "protoc", "", `Path to the protoc compiler.
+If given, we will call into this executable to generate the code; otherwise, we will call
+into "buf protoc".`)
+	cmdHTTP.Flags().StringVar(&protocFlags, "protoc-flags", "",
+		"Whitespace-separated list of flags to pass to {buf} protoc. This should include the list of input sources.")
 	cmdHTTP.Flags().StringVar(&genDocPath, "gendoc", "protoc-gen-doc", "Path to protoc-gen-doc binary.")
 	cmdHTTP.Flags().StringVar(&outPath, "out", "docs/generated/http", "File output path.")
 
@@ -60,7 +65,7 @@ var singleMethods = []string{
 // files. A full.md file is produced with all endpoints. The singleMethods
 // string slice is used to produce additional markdown files with a single
 // method per file.
-func runHTTP(genDocPath, protobufPath, outPath string) error {
+func runHTTP(protocPath, genDocPath, protocFlags, outPath string) error {
 	// Extract out all the data into a JSON file. We will use this JSON
 	// file to then generate full and single pages.
 	if err := os.MkdirAll(outPath, 0777); err != nil {
@@ -79,19 +84,21 @@ func runHTTP(genDocPath, protobufPath, outPath string) error {
 	defer func() {
 		_ = os.RemoveAll(tmpJSON)
 	}()
-	args := []string{"protoc",
+	var args []string
+	if len(protocPath) == 0 {
+		args = append(args, "protoc")
+	}
+	args = append(args,
 		fmt.Sprintf("--doc_out=%s", tmpJSON),
 		fmt.Sprintf("--doc_opt=%s,http.json", jsonTmpl),
-		fmt.Sprintf("--plugin=protoc-gen-doc=%s", genDocPath),
-	}
-	args = append(args, strings.Fields(protobufPath)...)
+		fmt.Sprintf("--plugin=protoc-gen-doc=%s", genDocPath))
+	args = append(args, strings.Fields(protocFlags)...)
 	// Generate the JSON file.
-	args = append(args,
-		"./pkg/server/serverpb/status.proto",
-		"./pkg/server/serverpb/admin.proto",
-		"./pkg/server/status/statuspb/status.proto",
-	)
-	cmd := exec.Command("buf", args...)
+	executable := protocPath
+	if len(protocPath) == 0 {
+		executable = "buf"
+	}
+	cmd := exec.Command(executable, args...)
 	if out, err := cmd.CombinedOutput(); err != nil {
 		fmt.Println(string(out))
 		return err

pkg/server/serverpb/admin.proto

@@ -748,7 +748,6 @@ message ChartCatalogResponse {
 
 // CARequest requests the CA cert anchoring this service.
 message CARequest {
-  // No information needed.
 }
 
 // CAResponse contains a PEM encoded copy of the CA cert for this service.