[BACKPORT pg15-cherrypicks] all: Bulk port from master - 46

Summary:
 f8e73e9 [#18192] YSQL: Introduce interface to mock tserver response in MiniCluster tests
 4ae68f4 Build break fix for centos7
 Excluded: 2ec9224 [#23033] Allow running YSQL upgrade unit tests with snapshot other than 2.0.9.0
 37912f1 [#22058] docdb: Disable connections on cloned db until cloning is complete
 059b855 [#22908] xCluster: Use XClusterRemoteClient across XCluster
 5dc5ee7 [#22849] YSQL: Correctly handle reset phase timeout errors in YSQL Connection Manager
 af49a1e [#22876][#22835][#22773] CDCSDK: Add new auto flag to identify non-eligible tables in CDC stream
 f3c4e14 [PLAT-14524] Up-version pekko to fix TLSActor infinite loop
 9388aea [#23052] yugabyted:  Restarting a node fails when data_dir is missing in user specified configuration.
 5cf9736 [PLAT-12685]: Generate a YBA metric for xcluster config table status.
 73fc90a [PLAT-14497]: Fix incremental backup time when none full backup exists
 e9b5ba5 [PLAT-14533]: Modify the gflags metadata support db version check
 8dca952 [PLAT-14432][Platform] Show certificate Database Node Certificate/key and Client Certificate/key for CA certs in certificate details modal
 6551e45 Add utkarsh.munjal to contributors.md
 bafa1cb [#21751] YSQL, ASH: Sampling of wait events

Test Plan: Jenkins: rebase: pg15-cherrypicks

Reviewers: jason, tfoucher

Subscribers: yql

Tags: #jenkins-ready

Differential Revision: https://phorge.dev.yugabyte.com/D36325

CONTRIBUTORS.md

@@ -162,3 +162,4 @@ This is a list of people who have contributed code to the [YugabyteDB](https://g
 * [asrinivasanyb](https://github.com/asrinivasanyb)
 * [devansh-ism](https://github.com/devansh-ism)
 * [swapshivam3](https://github.com/swapshivam3)
+* [utkarsh-um-yb] (https://github.com/utkarsh-um-yb)

bin/yugabyted

@@ -5145,7 +5145,7 @@ class ControlScript(object):
             user_configs = Configs.parse_user_config_file(args.config)
             # User should not be able to override data_dir
             if (self.configs.saved_data.get("cluster_member")
-                                    and args.data_dir is None):
+                and args.data_dir is None and user_configs.get("data_dir")):
                 user_configs_data_dir = [
                     path for path in user_configs.get("data_dir").split(',')
                 ]

java/yb-pgsql/src/test/java/org/yb/pgsql/TestYbAsh.java

@@ -19,9 +19,14 @@
 import java.sql.ResultSet;
 import java.sql.Statement;
 
+import java.util.ArrayList;
 import java.util.Collections;
-import java.util.concurrent.TimeUnit;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -280,4 +285,50 @@ public void testCatalogRequests() throws Exception {
       assertGreaterThan(res1, 0);
     }
   }
+
+  /**
+   * Test that we don't capture more than 'ysql_yb_ash_sample_size' number of samples
+   */
+  @Test
+  public void testSampleSize() throws Exception {
+    final int sample_size = 3;
+    setAshConfigAndRestartCluster(ASH_SAMPLING_INTERVAL, sample_size);
+    try (Statement statement = connection.createStatement()) {
+      statement.execute("CREATE TABLE test_table(k INT, v TEXT)");
+    }
+    final int NUM_THREADS = 5;
+    final int NUM_INSERTS_PER_THREAD = 100;
+    ExecutorService ecs = Executors.newFixedThreadPool(NUM_THREADS);
+    List<Future<?>> futures = new ArrayList<>();
+    for (int i = 1; i <= NUM_THREADS; ++i) {
+      final int threadIndex = i;
+      Future<?> future = ecs.submit(() -> {
+        try (Statement statement = connection.createStatement()) {
+          for (int j = 0; j < NUM_INSERTS_PER_THREAD; ++j) {
+            statement.execute(String.format("INSERT INTO test_table VALUES(%d, 'v-%d')",
+                threadIndex, j));
+          }
+        } catch (Exception e) {
+          fail(e.getMessage());
+        }
+      });
+      futures.add(future);
+    }
+    for (Future<?> future : futures) {
+      future.get();
+    }
+    ecs.shutdown();
+    ecs.awaitTermination(30, TimeUnit.SECONDS);
+    try (Statement statement = connection.createStatement()) {
+      ResultSet rs = statement.executeQuery("SELECT sample_time, wait_event_component, " +
+          "count(*) FROM " + ASH_VIEW + " GROUP BY sample_time, wait_event_component");
+      while (rs.next()) {
+        assertLessThanOrEqualTo(rs.getLong("count"), Long.valueOf(sample_size));
+      }
+      rs = statement.executeQuery("SELECT sample_weight FROM " + ASH_VIEW);
+      while (rs.next()) {
+        assertGreaterThanOrEqualTo(rs.getDouble("sample_weight"), Double.valueOf(1.0));
+      }
+    }
+  }
 }

managed/.sbtopts

@@ -1,6 +1,6 @@
 -J-XX:MinHeapFreeRatio=10
 -J-XX:MaxHeapFreeRatio=20
--J-Xmx2g
+-J-Xmx3g
 -J-XX:MaxMetaspaceSize=1g
 -J-XX:MetaspaceSize=500m
 -Dlogback.configurationFile=logback-sbt.xml

managed/build.sbt

@@ -203,7 +203,7 @@ libraryDependencies ++= Seq(
   "org.pac4j" %% "play-pac4j" % "9.0.2",
   "org.pac4j" % "pac4j-oauth" % "4.5.7" exclude("commons-io" , "commons-io"),
   "org.pac4j" % "pac4j-oidc" % "4.5.7" exclude("commons-io" , "commons-io"),
-  "org.playframework" %% "play-json" % "3.0.1",
+  "org.playframework" %% "play-json" % "3.0.4",
   "commons-validator" % "commons-validator" % "1.8.0",
   "org.apache.velocity" % "velocity-engine-core" % "2.3",
   "com.fasterxml.woodstox" % "woodstox-core" % "6.4.0",
@@ -968,7 +968,28 @@ dependencyOverrides += "jakarta.annotation" % "jakarta.annotation-api" % "1.3.5"
 dependencyOverrides += "jakarta.ws.rs" % "jakarta.ws.rs-api" % "2.1.6" % Test
 dependencyOverrides += "com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % "2.10.1" % Test
 
-val jacksonVersion         = "2.15.3"
+// This is a custom version, built based on 1.0.3 with the following commit added on top:
+// https://github.com/apache/pekko/commit/1e41829bf7abeec268b9a409f35051ed7f4e0090.
+// This is required to fix TLS infinite loop issue, which causes high CPU usage.
+// We can't use 1.1.0-M1 version yet, as it has the following issue:
+// https://github.com/playframework/playframework/pull/12662
+// Once the issue is fixed we should migrate back on stable version.
+val pekkoVersion         = "1.0.3-tls-loop-fix"
+
+val pekkoLibs = Seq(
+  "org.apache.pekko" %% "pekko-actor-typed",
+  "org.apache.pekko" %% "pekko-actor",
+  "org.apache.pekko" %% "pekko-protobuf-v3",
+  "org.apache.pekko" %% "pekko-serialization-jackson",
+  "org.apache.pekko" %% "pekko-slf4j",
+  "org.apache.pekko" %% "pekko-stream",
+)
+
+val pekkoOverrides = pekkoLibs.map(_ % pekkoVersion)
+
+dependencyOverrides ++= pekkoOverrides
+
+val jacksonVersion         = "2.17.1"
 
 val jacksonLibs = Seq(
   "com.fasterxml.jackson.core"       % "jackson-core",
@@ -1107,9 +1128,6 @@ val swaggerGenTest: TaskKey[Unit] = taskKey[Unit](
   "test generate swagger.json"
 )
 
-val swaggerJacksonVersion = "2.11.1"
-val swaggerJacksonOverrides = jacksonLibs.map(_ % swaggerJacksonVersion)
-
 lazy val swagger = project
   .dependsOn(root % "compile->compile;test->test")
   .settings(commonSettings)
@@ -1121,7 +1139,8 @@ lazy val swagger = project
       "com.github.dwickern" %% "swagger-play3.0" % "4.0.0"
     ),
 
-    dependencyOverrides ++= swaggerJacksonOverrides,
+    dependencyOverrides ++= pekkoOverrides,
+    dependencyOverrides ++= jacksonOverrides,
     dependencyOverrides += "org.scala-lang.modules" %% "scala-xml" % "2.1.0",
 
     swaggerGen := Def.taskDyn {

managed/devops/pex/Dockerfile

@@ -1,6 +1,12 @@
 # Get the latest docker image
 FROM quay.io/pypa/manylinux2014_x86_64
 
+
+RUN sed -i 's/mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/CentOS-*
+RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' \
+            /etc/yum.repos.d/CentOS-*
+
+
 # Perform general yum updates
 RUN yum --enablerepo=extras -y install epel-release python3-pip
 RUN pip3 install --upgrade pip

managed/devops/yb_release

@@ -28,9 +28,9 @@ activate_virtualenv
 bin/install_ansible_requirements.sh --force
 
 # Python modules build for yugabundle.
-cd "$yb_devops_home"
-docker build -t "$DOCKER_VENV_IMAGE_NAME" .
-docker run -v "$yb_devops_home:/devops" -u "$UID:$(id -g $UID)" "$DOCKER_VENV_IMAGE_NAME"
+# cd "$yb_devops_home"
+# docker build -t "$DOCKER_VENV_IMAGE_NAME" .
+# docker run -v "$yb_devops_home:/devops" -u "$UID:$(id -g $UID)" "$DOCKER_VENV_IMAGE_NAME"
 
 # PEX virtual env build for other deployments.
 cd "$yb_devops_home/pex"

managed/devops/yb_release_manifest.json

@@ -6,7 +6,6 @@
     "configure-cluster-server.yml",
     "destroy-instance.yml",
     "python3_requirements_frozen.txt",
-    "python*_modules.tar.gz",
     "preprovision.yml",
     "use_custom_ssh_port.yml",
     "yb-otel-collector.yml",

managed/project/plugins.sbt

@@ -22,7 +22,7 @@ dependencyOverrides += "com.google.googlejavaformat" % "google-java-format" % "1
 libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.4.14"
 
 // The Play plugin
-addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.0")
+addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.4")
 
 // ORM
 addSbtPlugin("org.playframework" % "sbt-play-ebean" % "8.0.0")

managed/src/main/java/MainModule.java

@@ -25,7 +25,7 @@
 import com.yugabyte.yw.commissioner.SupportBundleCleanup;
 import com.yugabyte.yw.commissioner.TaskExecutor;
 import com.yugabyte.yw.commissioner.TaskGarbageCollector;
-import com.yugabyte.yw.commissioner.XClusterSyncScheduler;
+import com.yugabyte.yw.commissioner.XClusterScheduler;
 import com.yugabyte.yw.commissioner.YbcUpgrade;
 import com.yugabyte.yw.common.AccessKeyRotationUtil;
 import com.yugabyte.yw.common.AccessManager;
@@ -220,7 +220,7 @@ public void configure() {
     bind(AccessKeyRotationUtil.class).asEagerSingleton();
     bind(GcpEARServiceUtil.class).asEagerSingleton();
     bind(YbcUpgrade.class).asEagerSingleton();
-    bind(XClusterSyncScheduler.class).asEagerSingleton();
+    bind(XClusterScheduler.class).asEagerSingleton();
     bind(PerfAdvisorScheduler.class).asEagerSingleton();
     bind(PermissionUtil.class).asEagerSingleton();
     bind(RoleUtil.class).asEagerSingleton();