Merge #95632 #95702 #96155

95632: obsservice: turn the ObsService into an OTLP server  r=andreimatei a=andreimatei

This patch makes the Obs Service implement the OTLP Logs gRPC service.
This reverses the connection direction for exporting events out of CRDB:
now CRDB connects out to the Obs Service through a new --obsservice-addr
flag. The Obs Service gets a new `--otlp-addr` flag taking the
host:port on which to expose the service.

By using standard OTLP we can have the data flow through an
OpenTelemetry Collector before going to the Obs Service. This makes the
Obs Service topology more flexible (since we can rely on the collector
to be deployed widely and funnel the data), and gives the collector the
opportunity to tee the data to other destinations.

Release note: None
Epic: CRDB-23641

95702: server,sql: add an interface for internal SQL sessions r=andreimatei a=andreimatei

This patch adds a way for a CRDB node to open pgwire connections to itself. This is implemented through the use of net.Conn's implemented by in-memory pipes.

I want to use this in particular in order to embed the Obs Service into CRBD. The Obs Service is a library that wants to be given a handle to the backend database in the form of a pgx connection pool. Now such a connection pool can be created based on this new "network" interface, without any network actually being involved.  Beyond that, I think this is a generally useful feature that we should have had for a long time - frequently it's useful internally to have a SQL session against the cluster; so far we've been doing that with the InternalExecutor, except that guy doesn't actually support sessions (it mostly supports single statements).

Release note: None
Epic: None

96155: sql: add a SHOW CLUSTER SETTING sql.defaults deprecation notice r=rafiss a=mastersobg

Release note (sql change): The `ALTER ROLE` syntax` allows users to set default values for session variables making `SET CLUSTER SETTINGS sql.defaults...` redundant. This PR adds a notice to `SHOW CLUSTER SETTING sql.defaults...` that directs the user to use the `ALTER ROLE` syntax instead.

Fixes: #88520

Co-authored-by: Andrei Matei <[email protected]>
Co-authored-by: Ivan Gorbachev <[email protected]>

pkg/base/test_server_args.go

@@ -164,6 +164,10 @@ type TestServerArgs struct {
 	// CockroachDB upgrades and periodically reports diagnostics to
 	// Cockroach Labs. Should remain disabled during unit testing.
 	StartDiagnosticsReporting bool
+
+	// ObsServiceAddr is the address to which events will be exported over OTLP.
+	// If empty, exporting events is inhibited.
+	ObsServiceAddr string
 }
 
 // TestClusterArgs contains the parameters one can set when creating a test

pkg/cli/cliflags/flags.go

@@ -1703,6 +1703,14 @@ commands, WARNING for client commands.`,
 		Description: `--sql-audit-dir=XXX is an alias for --log='sinks: {file-groups: {sql-audit: {channels: SENSITIVE_ACCESS, dir: ...}}}'.`,
 	}
 
+	ObsServiceAddr = FlagInfo{
+		Name:   "obsservice-addr",
+		EnvVar: "",
+		Description: `Address of an OpenTelemetry OTLP sink such as the
+Observability Service or the OpenTelemetry Collector. If set, telemetry
+events are exported to this address.`,
+	}
+
 	BuildTag = FlagInfo{
 		Name: "build-tag",
 		Description: `

pkg/cli/flags.go

@@ -541,6 +541,10 @@ func init() {
 	telemetryEnabledCmds := append(serverCmds, demoCmd, statementBundleRecreateCmd)
 	telemetryEnabledCmds = append(telemetryEnabledCmds, demoCmd.Commands()...)
 	for _, cmd := range telemetryEnabledCmds {
+		f := cmd.Flags()
+		cliflagcfg.StringFlag(f, &serverCfg.ObsServiceAddr, cliflags.ObsServiceAddr)
+		_ = f.MarkHidden(cliflags.ObsServiceAddr.Name)
+
 		// Report flag usage for server commands in telemetry. We do this
 		// only for server commands, as there is no point in accumulating
 		// telemetry if there's no telemetry reporting loop being started.

pkg/cli/start.go

@@ -333,6 +333,9 @@ type serverStartupInterface interface {
 
 	// AcceptClients starts listening for incoming SQL clients over the network.
 	AcceptClients(ctx context.Context) error
+	// AcceptInternalClients starts listening for incoming internal SQL clients over the
+	// loopback interface.
+	AcceptInternalClients(ctx context.Context) error
 
 	// InitialStart returns whether this node is starting for the first time.
 	// This is (currently) used when displaying the server status report
@@ -708,6 +711,11 @@ func createAndStartServerAsync(
 			// all these startup steps to fail. So we do not need to look at
 			// the "shutdown status" in serverStatusMu any more.
 
+			// Accept internal clients early, as RunInitialSQL might need it.
+			if err := s.AcceptInternalClients(ctx); err != nil {
+				return err
+			}
+
 			// Run one-off cluster initialization.
 			if err := s.RunInitialSQL(ctx, startSingleNode, "" /* adminUser */, "" /* adminPassword */); err != nil {
 				return err

pkg/gen/protobuf.bzl

@@ -45,7 +45,6 @@ PROTOBUF_SRCS = [
     "//pkg/obsservice/obspb/opentelemetry-proto/common/v1:v1_go_proto",
     "//pkg/obsservice/obspb/opentelemetry-proto/logs/v1:v1_go_proto",
     "//pkg/obsservice/obspb/opentelemetry-proto/resource/v1:v1_go_proto",
-    "//pkg/obsservice/obspb:obs_go_proto",
     "//pkg/repstream/streampb:streampb_go_proto",
     "//pkg/roachpb:roachpb_go_proto",
     "//pkg/rpc:rpc_go_proto",

pkg/obs/BUILD.bazel

@@ -10,20 +10,23 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/obs",
     visibility = ["//visibility:public"],
     deps = [
-        "//pkg/base",
         "//pkg/obsservice/obspb",
+        "//pkg/obsservice/obspb/opentelemetry-proto/collector/logs/v1:logs_service",
         "//pkg/obsservice/obspb/opentelemetry-proto/common/v1:common",
         "//pkg/obsservice/obspb/opentelemetry-proto/logs/v1:logs",
         "//pkg/obsservice/obspb/opentelemetry-proto/resource/v1:resource",
         "//pkg/util/log",
         "//pkg/util/mon",
+        "//pkg/util/netutil/addr",
         "//pkg/util/stop",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
+        "//pkg/util/tracing",
         "//pkg/util/uuid",
         "@com_github_cockroachdb_errors//:errors",
-        "@com_github_cockroachdb_redact//:redact",
-        "@org_golang_google_grpc//peer",
+        "@com_github_cockroachdb_logtags//:logtags",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//credentials/insecure",
     ],
 )