[#9797] YSQL: Merge pg15 branch into master

Switch YugabyteDB YSQL layer from PostgreSQL 11 to PostgreSQL 15.

.arclint

@@ -77,6 +77,7 @@
       "type": "text",
       "text.max-line-length": 100,
       "exclude": [
+        "(^pg15_tests/.*[.](sh|tsv)$)",
         "(^managed/.*[.]go$)",
         "(^managed/.*/go[.]mod)",
         "(^managed/.*/go[.]sum)",

java/yb-client/src/test/java/org/yb/minicluster/Metrics.java

@@ -163,11 +163,16 @@ public static class YSQLStat extends Metric {
      public final String query;
 
      public final long calls;
-     public final double total_time;
-     public final double min_time;
-     public final double max_time;
-     public final double mean_time;
-     public final double stddev_time;
+     public final double total_exec_time;
+     public final double total_plan_time;
+     public final double min_exec_time;
+     public final double min_plan_time;
+     public final double max_exec_time;
+     public final double max_plan_time;
+     public final double mean_exec_time;
+     public final double mean_plan_time;
+     public final double stddev_exec_time;
+     public final double stddev_plan_time;
      public final long rows;
      public final long local_blks_hit;
      public final long local_blks_read;
@@ -191,11 +196,16 @@ public static class YSQLStat extends Metric {
       query = metric.get("query").getAsString();
 
       calls = metric.get("calls").getAsLong();
-      total_time = metric.get("total_time").getAsDouble();
-      min_time = metric.get("min_time").getAsDouble();
-      max_time = metric.get("max_time").getAsDouble();
-      mean_time = metric.get("mean_time").getAsDouble();
-      stddev_time = metric.get("stddev_time").getAsDouble();
+      total_exec_time = metric.get("total_exec_time").getAsDouble();
+      total_plan_time = metric.get("total_plan_time").getAsDouble();
+      min_exec_time = metric.get("min_exec_time").getAsDouble();
+      min_plan_time = metric.get("min_plan_time").getAsDouble();
+      max_exec_time = metric.get("max_exec_time").getAsDouble();
+      max_plan_time = metric.get("max_plan_time").getAsDouble();
+      mean_exec_time = metric.get("mean_exec_time").getAsDouble();
+      mean_plan_time = metric.get("mean_plan_time").getAsDouble();
+      stddev_exec_time = metric.get("stddev_exec_time").getAsDouble();
+      stddev_plan_time = metric.get("stddev_plan_time").getAsDouble();
       rows = metric.get("rows").getAsLong();
       local_blks_hit = metric.get("local_blks_hit").getAsLong();
       local_blks_read = metric.get("local_blks_read").getAsLong();

java/yb-pgsql/src/test/java/org/yb/pgsql/BasePgRegressTest.java

@@ -25,6 +25,13 @@ public class BasePgRegressTest extends BasePgSQLTest {
   private static final String MASTERS_FLAG = "FLAGS_pggate_master_addresses";
   private static final String YB_ENABLED_IN_PG_ENV_VAR_NAME = "YB_ENABLED_IN_POSTGRES";
 
+  @Override
+  protected Map<String, String> getTServerFlags() {
+    Map<String, String> flagMap = super.getTServerFlags();
+    appendToYsqlPgConf(flagMap, "compute_query_id=regress");
+    return flagMap;
+  }
+
   public void runPgRegressTest(
       File inputDir, String schedule, long maxRuntimeMillis, File executable) throws Exception {
     final int tserverIndex = 0;

java/yb-pgsql/src/test/java/org/yb/pgsql/BasePgRegressTestPorted.java

@@ -0,0 +1,32 @@
+// Copyright (c) YugabyteDB, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied.  See the License for the specific language governing permissions and limitations
+// under the License.
+//
+package org.yb.pgsql;
+
+import java.util.Map;
+
+/**
+ * Regress test schedules should either contain only YB original tests or upstream PG ported tests.
+ * Subclass this class in case of ported tests in order to provide a similar environment to upstream
+ * PG and reduce differences with upstream's expectations.  At the time of writing (2024-07-02),
+ * some schedules hav a mix of test types, and the subclassing has yet to be applied for certain
+ * schedules.
+ */
+public class BasePgRegressTestPorted extends BasePgRegressTest {
+  @Override
+  protected Map<String, String> getTServerFlags() {
+    Map<String, String> flagMap = super.getTServerFlags();
+    flagMap.put("TEST_generate_ybrowid_sequentially", "true");
+    appendToYsqlPgConf(flagMap, "yb_use_hash_splitting_by_default=false");
+    return flagMap;
+  }
+}

java/yb-pgsql/src/test/java/org/yb/pgsql/BasePgSQLTest.java

@@ -222,15 +222,6 @@ protected static enum ConnectionManagerWarmupMode {
   // required.
   protected static final int MAX_ATTEMPTS_TO_DESTROY_CONTROL_CONN = 3;
 
-  protected static Map<String, String> FailOnConflictTestGflags = new HashMap<String, String>()
-    {
-      {
-          put("enable_wait_queues", "false");
-          // The retries are set to 2 to speed up the tests.
-          put("ysql_pg_conf_csv", maxQueryLayerRetriesConf(2));
-      }
-    };
-
   protected static ConcurrentSkipListSet<Integer> stuckBackendPidsConcMap =
       new ConcurrentSkipListSet<>();
 
@@ -279,6 +270,26 @@ protected Integer getYsqlRequestLimit() {
     return null;
   }
 
+  /**
+   * Add ysql_pg_conf_csv flag values using this method to avoid clobbering existing values.
+   * @param flagMap the map of flags to mutate
+   * @param value the raw text to append to the ysql_pg_conf_csv flag value
+   */
+  protected static void appendToYsqlPgConf(Map<String, String> flagMap, String value) {
+    final String flagName = "ysql_pg_conf_csv";
+    if (flagMap.containsKey(flagName)) {
+      flagMap.put(flagName, flagMap.get(flagName) + "," + value);
+    } else {
+      flagMap.put(flagName, value);
+    }
+  }
+
+  protected static void setFailOnConflictFlags(Map<String, String> flagMap) {
+    flagMap.put("enable_wait_queues", "false");
+    // The retries are set to 2 to speed up the tests.
+    appendToYsqlPgConf(flagMap, maxQueryLayerRetriesConf(2));
+  }
+
   /**
    * @return flags shared between tablet server and initdb
    */
@@ -382,6 +393,7 @@ public void initPostgresBefore() throws Exception {
     }
 
     connection = createTestRole();
+    allowSchemaPublic();
     pgInitialized = true;
   }
 
@@ -396,6 +408,12 @@ protected Connection createTestRole() throws Exception {
     return getConnectionBuilder().connect();
   }
 
+  private void allowSchemaPublic() throws Exception {
+    try (Statement statement = connection.createStatement()) {
+      statement.execute("GRANT ALL ON SCHEMA public TO public");
+    }
+  }
+
   public void restartClusterWithFlags(
       Map<String, String> additionalMasterFlags,
       Map<String, String> additionalTserverFlags) throws Exception {
@@ -474,6 +492,10 @@ public void cleanUpAfter() throws Exception {
       // TODO(dmitry): Workaround for #1721, remove after fix.
       stmt.execute("ROLLBACK");
       stmt.execute("DISCARD TEMP");
+
+      // TODO(tim): Workaround for DB-11127, remove after fix.
+      stmt.execute("RESET enable_seqscan");
+      stmt.execute("SET enable_bitmapscan = false");
     }
 
     cleanUpCustomDatabases();
@@ -708,7 +730,7 @@ protected AggregatedValue getStatementStat(String statName) throws Exception {
       YSQLStat ysqlStat = new Metrics(obj, true).getYSQLStat(statName);
       if (ysqlStat != null) {
         value.count += ysqlStat.calls;
-        value.value += ysqlStat.total_time;
+        value.value += ysqlStat.total_exec_time;
         value.rows += ysqlStat.rows;
       }
       scanner.close();
@@ -1661,7 +1683,6 @@ protected void isPartitionedOrderedIndexScan(Statement stmt,
     throws SQLException {
 
     String query_plan = getQueryPlanString(stmt, query);
-    assertTrue(query_plan.contains("Merge Append"));
     assertTrue(query_plan.contains("Index Scan using " + index));
   }
 
@@ -1675,7 +1696,6 @@ protected void isPartitionedOrderedIndexOnlyScan(Statement stmt,
     throws SQLException {
 
     String query_plan = getQueryPlanString(stmt, query);
-    assertTrue(query_plan.contains("Merge Append"));
     assertTrue(query_plan.contains("Index Only Scan using " + index));
   }
 
@@ -2283,9 +2303,7 @@ protected long getNumDocdbRequests(Statement stmt, String query) throws Exceptio
 
   /** Creates a new tserver and returns its id. **/
   protected int spawnTServer() throws Exception {
-    int tserverId = miniCluster.getNumTServers();
-    miniCluster.startTServer(getTServerFlags());
-    return tserverId;
+    return spawnTServerWithFlags(new HashMap<String, String>());
   }
 
   /** Creates a new tserver with additional flags and returns its id. **/

java/yb-pgsql/src/test/java/org/yb/pgsql/TestAlterTableWithConcurrentTxn.java

@@ -714,7 +714,7 @@ private void runAlterWithRollbackToEnsureNoTableDeletion() throws Exception {
       stmt.execute("INSERT INTO " + tableName + " VALUES (1, 1)");
 
       runInvalidQuery(stmt, "ALTER TABLE " + tableName + " ADD COLUMN c INT NOT NULL",
-          "column \"c\" contains null values");
+          "column \"c\" of relation \"" + tableName + "\" contains null values");
 
       Thread.sleep(20000); // 20 seconds
       assertQuery(stmt, "SELECT COUNT(*) FROM " + tableName, new Row(1));

java/yb-pgsql/src/test/java/org/yb/pgsql/TestBatchCopyFrom.java

@@ -46,7 +46,7 @@ public class TestBatchCopyFrom extends BasePgSQLTest {
   private static final String INVALID_NUM_SKIPPED_ROWS_ERROR_MSG =
       "argument to option \"skip\" must be a nonnegative integer";
   private static final String INVALID_COPY_INPUT_ERROR_MSG =
-      "invalid input syntax for integer";
+      "invalid input syntax for type integer";
   private static final String INVALID_FOREIGN_KEY_ERROR_MSG =
       "violates foreign key constraint";
   private static final String BATCH_TXN_SESSION_VARIABLE_NAME =

java/yb-pgsql/src/test/java/org/yb/pgsql/TestHdr.java

@@ -241,34 +241,36 @@ private static int checkJsonArray(JsonArray jsonArray, double minTime, double ma
 
   private static void runCheckSleepQuery(Statement statement, int bucketFactor)
       throws SQLException {
-    ResultSet rs = statement.executeQuery("SELECT query, min_time, max_time, yb_latency_histogram "
+    ResultSet rs = statement.executeQuery("SELECT query, min_exec_time,"
+        + "max_exec_time, yb_latency_histogram "
         + "FROM pg_stat_statements WHERE query like '%select pg_sleep%'");
 
     assertTrue(rs.next());
-    double minTime = rs.getDouble(2);
-    double maxTime = rs.getDouble(3);
+    double minExecTime = rs.getDouble(2);
+    double maxExecTime = rs.getDouble(3);
     String latencyHistogram = rs.getString(4);
-    BucketInterval minInterval = getExpInterval(minTime, bucketFactor, false);
-    BucketInterval maxInterval = getExpInterval(maxTime, bucketFactor, false);
+    BucketInterval minInterval = getExpInterval(minExecTime, bucketFactor, false);
+    BucketInterval maxInterval = getExpInterval(maxExecTime, bucketFactor, false);
 
     JSONArray jsonArray = new JSONArray(latencyHistogram);
     JsonElement jsonPrint = JsonParser.parseString(latencyHistogram);
     LOG.info("Response:\n" + JsonUtil.asPrettyString(jsonPrint));
 
     if (minInterval.equals(maxInterval)) {
       assertEquals(1, jsonArray.length());
-      checkJSONObject(jsonArray.getJSONObject(0), 2, minTime, minInterval);
-      checkJSONObject(jsonArray.getJSONObject(0), 2, maxTime, maxInterval);
+      checkJSONObject(jsonArray.getJSONObject(0), 2, minExecTime, minInterval);
+      checkJSONObject(jsonArray.getJSONObject(0), 2, maxExecTime, maxInterval);
     } else {
       assertEquals(2, jsonArray.length());
-      checkJSONObject(jsonArray.getJSONObject(0), 1, minTime, minInterval);
-      checkJSONObject(jsonArray.getJSONObject(jsonArray.length() - 1), 1, maxTime, maxInterval);
+      checkJSONObject(jsonArray.getJSONObject(0), 1, minExecTime, minInterval);
+      checkJSONObject(jsonArray.getJSONObject(jsonArray.length() - 1), 1, maxExecTime, maxInterval);
     }
   }
 
   private static void runCheckHdrArray(Statement statement, int iterations, int bucketFactor)
       throws SQLException {
-    ResultSet rs = statement.executeQuery("SELECT query, min_time, max_time, yb_latency_histogram "
+    ResultSet rs = statement.executeQuery("SELECT query, min_exec_time,"
+        + "max_exec_time, yb_latency_histogram "
         + "FROM pg_stat_statements WHERE query like '%select pg_sleep%'");
 
     assertTrue(rs.next());
@@ -346,8 +348,8 @@ private static void verifyStatementHist(String statName, int iterations) throws
         JsonObject obj = tree.getAsJsonObject();
         YSQLStat ysqlStat = new Metrics(obj, true).getYSQLStat(statName);
         if (ysqlStat != null) {
-          double minTime = ysqlStat.min_time;
-          double maxTime = ysqlStat.max_time;
+          double minTime = ysqlStat.min_exec_time;
+          double maxTime = ysqlStat.max_exec_time;
           currIterations += checkJsonArray(ysqlStat.yb_latency_histogram, minTime, maxTime,
               DEFAULT_YB_HDR_BUCKET_FACTOR);
         }

java/yb-pgsql/src/test/java/org/yb/pgsql/TestOneOrTwoAdmins.java

@@ -37,7 +37,7 @@ protected Map<String, String> getTServerFlags() {
     // This test expects one of two conflicting transactions to be killed immediately in each
     // iteration. Therefore, we run it in fail-on-conflict concurrency control.
     // TODO(wait-queues): https://github.com/yugabyte/yugabyte-db/issues/17871
-    flagMap.putAll(FailOnConflictTestGflags);
+    setFailOnConflictFlags(flagMap);
     return flagMap;
   }
 

java/yb-pgsql/src/test/java/org/yb/pgsql/TestPg15Regress.java

@@ -0,0 +1,50 @@
+// Copyright (c) YugaByte, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied.  See the License for the specific language governing permissions and limitations
+// under the License.
+//
+package org.yb.pgsql;
+
+import java.util.Map;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.yb.util.BuildTypeUtil;
+import org.yb.util.YBTestRunnerNonTsanOnly;
+
+// Runs the pg_regress test suite on YB code.
+@RunWith(value = YBTestRunnerNonTsanOnly.class)
+public class TestPg15Regress extends BasePgRegressTest {
+    @Override
+    public int getTestMethodTimeoutSec() {
+        return BuildTypeUtil.nonSanitizerVsSanitizer(2100, 2700);
+    }
+
+  @Override
+  protected Map<String, String> getTServerFlags() {
+    Map<String, String> flagMap = super.getTServerFlags();
+    flagMap.put("allowed_preview_flags_csv", "ysql_yb_enable_replication_commands");
+    flagMap.put("ysql_yb_enable_replication_commands", "true");
+    return flagMap;
+  }
+
+  @Override
+  protected Map<String, String> getMasterFlags() {
+    Map<String, String> flagMap = super.getMasterFlags();
+    flagMap.put("allowed_preview_flags_csv", "ysql_yb_enable_replication_commands");
+    flagMap.put("ysql_yb_enable_replication_commands", "true");
+    return flagMap;
+  }
+
+    @Test
+    public void testPg15Regress() throws Exception {
+        runPgRegressTest("yb_pg15");
+    }
+}

java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgAlterTable.java

@@ -137,13 +137,13 @@ public void testAddColumnWithNotNullConstraint() throws Exception {
       runInvalidQuery(
           statement,
           "INSERT INTO test_table(id, b) VALUES(1, 3)",
-          "null value in column \"a\" violates not-null constraint"
+          "null value in column \"a\" of relation \"test_table\" violates not-null constraint"
       );
 
       runInvalidQuery(
           statement,
           "ALTER TABLE test_table ADD c int NOT NULL",
-          "column \"c\" contains null values"
+          "column \"c\" of relation \"test_table\" contains null values"
       );
     }
   }
@@ -203,7 +203,8 @@ public void testAddColumnWithCheckConstraint() throws Exception {
       runInvalidQuery(
           statement,
           "ALTER TABLE test_table ADD b int CHECK (b IS NOT NULL)",
-          "check constraint \"test_table_b_check\" is violated by some row"
+          ("check constraint \"test_table_b_check\" of relation \"test_table\" " +
+           "is violated by some row")
       );
     }
   }

java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgAuthorization.java

@@ -448,7 +448,7 @@ public void testAttributes() throws Exception {
         runInvalidQuery(
             statement,
             "ALTER ROLE su LOGIN",
-            "must be superuser to alter superusers"
+            "must be superuser to alter superuser roles or change superuser attribute"
         );
       });
 
@@ -3256,6 +3256,7 @@ public void testLongPasswords() throws Exception {
 
       // Create role with password.
       statement.execute("DROP ROLE IF EXISTS pass_role");
+      statement.execute("SET password_encryption = 'md5'");
       statement.execute(
           String.format("CREATE ROLE pass_role LOGIN PASSWORD '%s'", passwordWithLen5000));
 

java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgAutoExplain.java

@@ -72,7 +72,7 @@ protected Map<String, String> getTServerFlags() {
     Map<String, String> flagMap = super.getTServerFlags();
     flagMap.put("ysql_prefetch_limit", "1024");
     flagMap.put("ysql_session_max_batch_size", "512");
-    flagMap.put("ysql_pg_conf_csv", "\"shared_preload_libraries=auto_explain\"");
+    appendToYsqlPgConf(flagMap, "shared_preload_libraries=auto_explain");
     return flagMap;
   }