kvserver: implement closed ts side-transport publisher

The side-transport is a component running on each node and periodically
publishing closed timestamps on ranges with the lease on the respective
node. This complements the closing of timestamps through Raft commands
such that inactive ranges still have their timestamp advanced.

This commit introduces only the publishing side (the consumer is coming
separately) - the guy opening streaming connections to all other nodes
with follower replicas for some ranges with local leases and
periodically publishing closed timestamp updates on a bunch of ranges at
once.

Care has been taken to make the communication protocol efficient. Each
stream is stateful and the information in every message is nicely
compressed.

See [the RFC](#56675) for details.

Release justification: Needed for global tables.
Release note: None

pkg/BUILD.bazel

@@ -91,9 +91,11 @@ ALL_TESTS = [
     "//pkg/kv/kvserver/closedts/container:container_test",
     "//pkg/kv/kvserver/closedts/minprop:minprop_test",
     "//pkg/kv/kvserver/closedts/provider:provider_test",
+    "//pkg/kv/kvserver/closedts/sidetransport:sidetransport_test",
     "//pkg/kv/kvserver/closedts/storage:storage_test",
     "//pkg/kv/kvserver/closedts/tracker:tracker_test",
     "//pkg/kv/kvserver/closedts/transport:transport_test",
+    "//pkg/kv/kvserver/closedts:closedts_test",
     "//pkg/kv/kvserver/concurrency:concurrency_test",
     "//pkg/kv/kvserver/gc:gc_test",
     "//pkg/kv/kvserver/idalloc:idalloc_test",

pkg/kv/kvserver/BUILD.bazel

@@ -116,6 +116,7 @@ go_library(
         "//pkg/kv/kvserver/closedts",
         "//pkg/kv/kvserver/closedts/container",
         "//pkg/kv/kvserver/closedts/ctpb",
+        "//pkg/kv/kvserver/closedts/sidetransport",
         "//pkg/kv/kvserver/closedts/storage",
         "//pkg/kv/kvserver/closedts/tracker",
         "//pkg/kv/kvserver/concurrency",
@@ -244,6 +245,7 @@ go_test(
         "replica_application_cmd_buf_test.go",
         "replica_application_state_machine_test.go",
         "replica_batch_updates_test.go",
+        "replica_closedts_test.go",
         "replica_command_test.go",
         "replica_consistency_test.go",
         "replica_evaluate_test.go",

pkg/kv/kvserver/closedts/BUILD.bazel

@@ -1,9 +1,10 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "closedts",
     srcs = [
         "closedts.go",
+        "policy.go",
         "setting.go",
     ],
     importpath = "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts",
@@ -16,3 +17,16 @@ go_library(
         "@com_github_cockroachdb_errors//:errors",
     ],
 )
+
+go_test(
+    name = "closedts_test",
+    srcs = ["policy_test.go"],
+    embed = [":closedts"],
+    deps = [
+        "//pkg/roachpb",
+        "//pkg/util/hlc",
+        "//pkg/util/leaktest",
+        "//pkg/util/log",
+        "@com_github_stretchr_testify//require",
+    ],
+)

pkg/kv/kvserver/closedts/ctpb/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "client.go",
         "entry.go",
         "server.go",
+        "service.go",
     ],
     embed = [":ctpb_go_proto"],
     importpath = "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb",
@@ -24,6 +25,7 @@ proto_library(
     strip_import_prefix = "/pkg",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/roachpb:roachpb_proto",
         "//pkg/util/hlc:hlc_proto",
         "@com_github_gogo_protobuf//gogoproto:gogo_proto",
         "@go_googleapis//google/api:annotations_proto",
@@ -37,6 +39,7 @@ go_proto_library(
     proto = ":ctpb_proto",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/roachpb",
         "//pkg/util/hlc",
         "@com_github_gogo_protobuf//gogoproto",
         "@org_golang_google_genproto//googleapis/api/annotations:go_default_library",

pkg/kv/kvserver/closedts/ctpb/service.go

@@ -0,0 +1,14 @@
+// Copyright 2021 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package ctpb
+
+// SeqNum identifies a ctpb.Update.
+type SeqNum int64