From 0a164ed0032f9f0743d92c030ceaaff4027bd39d Mon Sep 17 00:00:00 2001 From: Yugabyte CI Date: Thu, 26 Sep 2024 12:14:49 -0400 Subject: [PATCH] [BACKPORT pg15-cherrypicks] all: Bulk port from master - 110 Summary: 35b12d27b4 [PLAT-15404] Average YSQL operations latency alert is using incorrect units (ms vs microsecs) Excluded: 008f88534c [#23788] YSQL, QueryDiagnostics: Fixing issues in pg_stat_statements when no query executed 6ca8cc4011 [#23810] yugabyted-ui: UI is displaying incorrect disk size when multiple data directories dca5923167 [PLAT-15034][K8s] Add changes to apply master_join_existing_cluster gflag fa9b370ea7 [docs] Update content for getting started page for CDC logical replication (#23916) 8db0ffbbff [PLAT-15380] clock drift alert did not reference nodes 44ae377264 [PLAT-15349] Mark universe update as success after update lb config Excluded: 9f9081921e [#24121] xCluster: Fix xcluster_outbound_replication_group-itest TestGetStreamByTableId 250a4d50d3 [#24026] docdb: Fix SIGSEGV from MaxPersistentOpId after flush 0d1046abdb [DEVOPS-3238] Move macOS build to macos13 (Ventura) 87cffc6991 [#24137] DocDB: Add gflag_allowlist to yb_release_manifest 678d2771de [#21178] docdb: Add metric for the max master follower heartbeat delay. ff97f51a1c [doc][ybm] Certificate links (#24139) Excluded: d26b62d0e9 [#21733] YSQL: ParallelAppend and pg_hint_plan 3ffe5a7cf8 [PLAT-10519]Lack of Client-Side Inactivity Timeout - Part 1 254e164718 [PLAT-15432] remove status,sizeInBytes from manifest.json file Test Plan: Jenkins: rebase: pg15-cherrypicks Reviewers: tfoucher, fizaa, telgersma Differential Revision: https://phorge.dev.yugabyte.com/D38454 --- .../drivers-orms/go/yb-pgx-reference.md | 2 +- .../drivers-orms/java/postgres-jdbc.md | 2 +- .../drivers-orms/java/yugabyte-jdbc.md | 2 +- .../nodejs/postgres-node-driver.md | 2 +- .../nodejs/postgres-pg-reference.md | 2 +- .../nodejs/yugabyte-node-driver.md | 2 +- .../drivers-orms/orms/nodejs/ysql-prisma.md | 2 +- .../drivers-orms/python/postgres-psycopg2.md | 2 +- .../drivers-orms/python/yugabyte-psycopg2.md | 2 +- .../rust/rust-postgres-reference.md | 2 +- .../using-logical-replication/get-started.md | 16 +++-- .../debezium-connector-yugabytedb.md | 2 +- .../preview/integrations/hashicorp-vault.md | 2 +- .../tutorials/azure/azure-api-management.md | 10 +-- .../tutorials/azure/azure-private-link.md | 2 +- .../drivers-orms/go/yb-pgx-reference.md | 4 +- .../stable/drivers-orms/java/postgres-jdbc.md | 2 +- .../stable/drivers-orms/java/yugabyte-jdbc.md | 2 +- .../nodejs/postgres-node-driver.md | 2 +- .../nodejs/postgres-pg-reference.md | 2 +- .../nodejs/yugabyte-node-driver.md | 2 +- .../drivers-orms/orms/nodejs/ysql-prisma.md | 2 +- .../drivers-orms/python/postgres-psycopg2.md | 2 +- .../drivers-orms/python/yugabyte-psycopg2.md | 2 +- .../rust/rust-postgres-reference.md | 2 +- .../using-logical-replication/get-started.md | 14 ++-- .../debezium-connector-yugabytedb.md | 2 +- jenkins_jobs.yml | 7 +- .../tasks/CreateKubernetesUniverse.java | 27 +++++++- .../tasks/CreateSupportBundle.java | 10 +-- .../yw/commissioner/tasks/CreateUniverse.java | 2 +- .../tasks/KubernetesTaskBase.java | 22 ++++--- .../tasks/UniverseDefinitionTaskBase.java | 9 ++- .../tasks/UpdateLoadBalancerConfig.java | 10 ++- .../subtasks/KubernetesCommandExecutor.java | 5 ++ .../yugabyte/yw/common/KubernetesUtil.java | 4 +- .../main/resources/alert/alert_templates.yml | 18 ++--- .../common/V378__Update_ClockDrift_Alert.sql | 2 + managed/src/main/resources/reference.conf | 5 +- .../tasks/CreateKubernetesUniverseTest.java | 25 ++++++- .../KubernetesCommandExecutorTest.java | 24 +++---- python/yugabyte/yb_dist_tests.py | 1 + .../tablet_health_manager-itest.cc | 65 ++++++++++++++++++- src/yb/master/catalog_manager.cc | 28 ++++++++ src/yb/master/catalog_manager.h | 2 + src/yb/tablet/tablet_peer.cc | 7 +- src/yb/util/monotime.cc | 6 ++ src/yb/util/monotime.h | 2 + yb_release_manifest.json | 1 + .../cmd/server/handlers/api_cluster.go | 44 +++++++++---- 50 files changed, 310 insertions(+), 106 deletions(-) create mode 100644 managed/src/main/resources/db/migration/default_/common/V378__Update_ClockDrift_Alert.sql diff --git a/docs/content/preview/drivers-orms/go/yb-pgx-reference.md b/docs/content/preview/drivers-orms/go/yb-pgx-reference.md index 78d0134eff6a..861532d433bc 100644 --- a/docs/content/preview/drivers-orms/go/yb-pgx-reference.md +++ b/docs/content/preview/drivers-orms/go/yb-pgx-reference.md @@ -239,7 +239,7 @@ For more details, see the [pgxpool package](https://pkg.go.dev/github.com/jackc/ To build a Go application that communicates securely over SSL with YugabyteDB database, you need the root certificate (`ca.crt`) of the YugabyteDB cluster. To generate these certificates and install them while launching the cluster, follow the instructions in [Create server certificates](../../../secure/tls-encryption/server-certificates/). -Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [CA certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [CA certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). For a YugabyteDB Aeon cluster, or a YugabyteDB cluster with SSL/TLS enabled, set the SSL-related environment variables as follows at the client side. diff --git a/docs/content/preview/drivers-orms/java/postgres-jdbc.md b/docs/content/preview/drivers-orms/java/postgres-jdbc.md index 668175720ec7..07c69ca6e906 100644 --- a/docs/content/preview/drivers-orms/java/postgres-jdbc.md +++ b/docs/content/preview/drivers-orms/java/postgres-jdbc.md @@ -146,7 +146,7 @@ String yburl = "jdbc:postgresql://hostname:port/database?user=yugabyte&password= Connection conn = DriverManager.getConnection(yburl); ``` -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). ### Step 3: Write your application diff --git a/docs/content/preview/drivers-orms/java/yugabyte-jdbc.md b/docs/content/preview/drivers-orms/java/yugabyte-jdbc.md index 8609236a5d87..56e2063312f8 100644 --- a/docs/content/preview/drivers-orms/java/yugabyte-jdbc.md +++ b/docs/content/preview/drivers-orms/java/yugabyte-jdbc.md @@ -176,7 +176,7 @@ jdbc:yugabytedb://hostname:port/database?user=yugabyte&password=yugabyte&load-ba ssl=true&sslmode=verify-full&sslrootcert=~/.postgresql/root.crt ``` -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). To use load balancing and SSL mode verify-full with a cluster in YugabyteDB Aeon, you need to provide the additional `sslhostnameverifier` parameter, set to `com.yugabyte.ysql.YBManagedHostnameVerifier`. (Available in driver version 42.3.5-yb-2 or later. For previous versions of the driver, use `verify-ca`.) diff --git a/docs/content/preview/drivers-orms/nodejs/postgres-node-driver.md b/docs/content/preview/drivers-orms/nodejs/postgres-node-driver.md index c3e43bb6d3c0..75d7d0298cd4 100644 --- a/docs/content/preview/drivers-orms/nodejs/postgres-node-driver.md +++ b/docs/content/preview/drivers-orms/nodejs/postgres-node-driver.md @@ -115,7 +115,7 @@ const config = { } ``` -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). Refer to [Configure SSL/TLS](../postgres-pg-reference/#configure-ssl-tls) for more information on node-postgresql default and supported SSL modes, and other examples for setting up your connection strings when using SSL. diff --git a/docs/content/preview/drivers-orms/nodejs/postgres-pg-reference.md b/docs/content/preview/drivers-orms/nodejs/postgres-pg-reference.md index 3b2ea4c49645..4be60715f735 100644 --- a/docs/content/preview/drivers-orms/nodejs/postgres-pg-reference.md +++ b/docs/content/preview/drivers-orms/nodejs/postgres-pg-reference.md @@ -144,7 +144,7 @@ client To build a Node.js application that communicates securely over SSL, get the root certificate (`ca.crt`) of the YugabyteDB Cluster. If certificates are not generated yet, follow the instructions in [Create server certificates](../../../secure/tls-encryption/server-certificates/). -Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [Download your cluster certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [Download your cluster certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). The node-postgres driver allows you to avoid including the parameters like `sslcert`, `sslkey`, `sslrootcert`, or `sslmode` in the connection string. You can pass the object which includes `connectionString` and `ssl` object which has various fields including the following: diff --git a/docs/content/preview/drivers-orms/nodejs/yugabyte-node-driver.md b/docs/content/preview/drivers-orms/nodejs/yugabyte-node-driver.md index 6e3735fce3d4..c2be085c550d 100644 --- a/docs/content/preview/drivers-orms/nodejs/yugabyte-node-driver.md +++ b/docs/content/preview/drivers-orms/nodejs/yugabyte-node-driver.md @@ -109,7 +109,7 @@ Refer to [Configure SSL/TLS](../postgres-pg-reference/#configure-ssl-tls) for mo #### Use SSL with YugabyteDB Aeon -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). With clusters in YugabyteDB Aeon, you can't use SSL mode verify-full; other SSL modes are supported. To use the equivalent of verify-full, don't set the `sslmode` or `sslrootcert` parameters in your connection string; instead, use the `ssl` object with the following parameters: diff --git a/docs/content/preview/drivers-orms/orms/nodejs/ysql-prisma.md b/docs/content/preview/drivers-orms/orms/nodejs/ysql-prisma.md index f9e875a9b140..a5c911cd17b7 100644 --- a/docs/content/preview/drivers-orms/orms/nodejs/ysql-prisma.md +++ b/docs/content/preview/drivers-orms/orms/nodejs/ysql-prisma.md @@ -93,7 +93,7 @@ DATABASE_URL="postgresql://:@:/" If you have a YugabyteDB Aeon cluster, do the following: -1. Download your [cluster certificate](../../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +1. Download your [cluster certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). 1. Install OpenSSL, if not present. diff --git a/docs/content/preview/drivers-orms/python/postgres-psycopg2.md b/docs/content/preview/drivers-orms/python/postgres-psycopg2.md index 6bc4c8d56cb8..a63c677694e5 100644 --- a/docs/content/preview/drivers-orms/python/postgres-psycopg2.md +++ b/docs/content/preview/drivers-orms/python/postgres-psycopg2.md @@ -129,7 +129,7 @@ The following is an example for connecting to YugabyteDB with SSL encryption ena conn = psycopg2.connect("host= port=5433 dbname=yugabyte user= password= sslmode=verify-full sslrootcert=/Users/my-user/Downloads/root.crt") ``` -If you have created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you have created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). ### Step 3: Write your application diff --git a/docs/content/preview/drivers-orms/python/yugabyte-psycopg2.md b/docs/content/preview/drivers-orms/python/yugabyte-psycopg2.md index 5a5d78258665..c4a79123097b 100644 --- a/docs/content/preview/drivers-orms/python/yugabyte-psycopg2.md +++ b/docs/content/preview/drivers-orms/python/yugabyte-psycopg2.md @@ -184,7 +184,7 @@ The following is an example for connecting to a YugabyteDB cluster with SSL enab conn = psycopg2.connect("host= port=5433 dbname=yugabyte user= password= load_balance=true sslmode=verify-full sslrootcert=/path/to/root.crt") ``` -The Yugabyte Psycopg2 smart driver does not support SSL mode verify-full for clusters in YugabyteDB Aeon. Use verify-ca or the upstream psycopg2 driver. If your cluster is on YugabyteDB Aeon, use the cluster credentials for user and password, and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +The Yugabyte Psycopg2 smart driver does not support SSL mode verify-full for clusters in YugabyteDB Aeon. Use verify-ca or the upstream psycopg2 driver. If your cluster is on YugabyteDB Aeon, use the cluster credentials for user and password, and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). ### Step 3: Write your application diff --git a/docs/content/preview/drivers-orms/rust/rust-postgres-reference.md b/docs/content/preview/drivers-orms/rust/rust-postgres-reference.md index 0af175540a58..4a0c210531d4 100644 --- a/docs/content/preview/drivers-orms/rust/rust-postgres-reference.md +++ b/docs/content/preview/drivers-orms/rust/rust-postgres-reference.md @@ -355,7 +355,7 @@ The following is an example connection URL for connecting to a YugabyteDB cluste "postgresql://127.0.0.1:5434/yugabyte?user=yugabyte&password=yugabyte&load_balance=true&sslmode=require" ``` -If you created a cluster on [YugabyteDB Aeon](../../../yugabyte-cloud/), use the cluster credentials and download the [SSL Root certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and download the [SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). The following is an example application for connecting to a YugabyteDB cluster with SSL enabled: diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/get-started.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/get-started.md index 8f8c8ec724a0..cb10be246bc7 100644 --- a/docs/content/preview/explore/change-data-capture/using-logical-replication/get-started.md +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/get-started.md @@ -173,7 +173,7 @@ To start the services needed for this tutorial, you must: - [Start Zookeeper](#start-zookeeper) - [Start Kafka](#start-kafka) -- [Start a YugabyteDB database](#start-a-yugabytedb-database) +- [Start YugabyteDB](#start-yugabytedb) - [Start Kafka Connect](#start-kafka-connect) #### Start Zookeeper @@ -202,9 +202,11 @@ In this tutorial, you will always connect to Kafka from in a Docker container. A {{< /note >}} -#### Start a YugabyteDB database +#### Start YugabyteDB -At this point, you have started Zookeeper and Kafka, but you still need a database server from which Debezium can capture changes. In this procedure, you start a YugabyteDB instance with an example database. Follow the [Quick Start](../../../../quick-start) to start an instance using yugabyted. +At this point, you have started Zookeeper and Kafka, but you still need a database server from which Debezium can capture changes. In this procedure, you start a YugabyteDB instance with an example database. The example uses sample data in SQL scripts that are included with your YugabyteDB installation in the `share` directory. + +Follow the [Quick Start](../../../../quick-start) to start an instance using yugabyted. {{< note title="Note" >}} @@ -237,7 +239,7 @@ After starting YugabyteDB, use ysqlsh to create your database: yugabyte=# ``` -1. Load the schema of the sample tables: +1. Load the schema of the sample tables. ```sql yugabyte=# \i share/schema.sql @@ -288,10 +290,10 @@ After starting YugabyteDB, you start the Kafka Connect service. This service exp 1. Open a new terminal, and use it to start the Kafka Connect service in a container. - The following command runs a new container using the `dz.2.5.2.yb.2024.1.SNAPSHOT.1` version of the `quay.io/yugabyte/ybdb-debezium` image: + The following command runs a new container using the `dz.2.5.2.yb.2024.1` version of the `quay.io/yugabyte/ybdb-debezium` image: ```sh - docker run -it --rm --name connect -p 8083:8083 -p 1976:1976 -e GROUP_ID=1 -e CONFIG_STORAGE_TOPIC=my_connect_configs -e OFFSET_STORAGE_TOPIC=my_connect_offsets -e STATUS_STORAGE_TOPIC=my_connect_statuses -e CLASSPATH=/kafka/connect/ --link zookeeper:zookeeper --link kafka:kafka quay.io/yugabyte/ybdb-debezium:dz.2.5.2.yb.2024.1.SNAPSHOT.1 + docker run -it --rm --name connect -p 8083:8083 -p 1976:1976 -e GROUP_ID=1 -e CONFIG_STORAGE_TOPIC=my_connect_configs -e OFFSET_STORAGE_TOPIC=my_connect_offsets -e STATUS_STORAGE_TOPIC=my_connect_statuses -e CLASSPATH=/kafka/connect/ --link zookeeper:zookeeper --link kafka:kafka quay.io/yugabyte/ybdb-debezium:dz.2.5.2.yb.2024.1 ``` 1. Verify that Kafka Connect started and is ready to accept connections. You should see output similar to the following: @@ -449,7 +451,7 @@ Using KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://172.17.0.7:9092 Using KAFKA_BROKER=172.17.0.3:9092 Contents of topic dbserver1.public.products: ... -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"before"},{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"name":"event.block","version":1,"field":"transaction"}],"optional":false,"name":"dbserver1.public.products.Envelope","version":1},"payload":{"before":null,"after":{"id":{"value":147,"set":true},"created_at":{"value":1500306107286000,"set":true},"category":{"value":"Doohickey","set":true},"ean":{"value":"6590063715","set":true},"price":{"value":44.4315141414441,"set":true},"quantity":{"value":5000,"set":true},"rating":{"value":4.6,"set":true},"title":{"value":"Mediocre Wool Toucan","set":true},"vendor":{"value":"Bradtke, Wilkinson and Reilly","set":true}},"source":{"version":"dz.2.5.2.yb.2024.1-SNAPSHOT","connector":"postgresql","name":"dbserver1","ts_ms":1721400304248,"snapshot":"true","db":"yugabyte","sequence":"[null,\"2\"]","schema":"public","table":"products","txId":2,"lsn":2,"xmin":null},"op":"r","ts_ms":1721400309609,"transaction":null}} +{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"before"},{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"name":"event.block","version":1,"field":"transaction"}],"optional":false,"name":"dbserver1.public.products.Envelope","version":1},"payload":{"before":null,"after":{"id":{"value":147,"set":true},"created_at":{"value":1500306107286000,"set":true},"category":{"value":"Doohickey","set":true},"ean":{"value":"6590063715","set":true},"price":{"value":44.4315141414441,"set":true},"quantity":{"value":5000,"set":true},"rating":{"value":4.6,"set":true},"title":{"value":"Mediocre Wool Toucan","set":true},"vendor":{"value":"Bradtke, Wilkinson and Reilly","set":true}},"source":{"version":"dz.2.5.2.yb.2024.1","connector":"postgresql","name":"dbserver1","ts_ms":1721400304248,"snapshot":"true","db":"yugabyte","sequence":"[null,\"2\"]","schema":"public","table":"products","txId":2,"lsn":2,"xmin":null},"op":"r","ts_ms":1721400309609,"transaction":null}} ... ``` diff --git a/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md index 9ac9510622fc..95a2426f74df 100644 --- a/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md +++ b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md @@ -1053,7 +1053,7 @@ If you have a YugabyteDB cluster with SSL enabled, you need to obtain the root c * [Local deployments](../../../../secure/tls-encryption/) * [YugabyteDB Anywhere](../../../../yugabyte-platform/security/enable-encryption-in-transit/#enable-encryption-in-transit) -* [YugabyteDB Aeon](../../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate) +* [YugabyteDB Aeon](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate) {{< /note >}} diff --git a/docs/content/preview/integrations/hashicorp-vault.md b/docs/content/preview/integrations/hashicorp-vault.md index 5538a1368fb2..16fe7e87fcd2 100644 --- a/docs/content/preview/integrations/hashicorp-vault.md +++ b/docs/content/preview/integrations/hashicorp-vault.md @@ -164,7 +164,7 @@ vault lease revoke To allow YSQL Hashicorp Vault plugin to communicate securely over SSL with YugabyteDB database, you need the root certificate (`ca.crt`) of the YugabyteDB cluster. To generate these certificates and install them while launching the cluster, follow the instructions in [Create server certificates](../../secure/tls-encryption/server-certificates/). -Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [CA certificate](../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [CA certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). To start a secure local YugabyteDB cluster using `yugabyted`, refer to [Create a local multi-node cluster](../../reference/configuration/yugabyted/#create-a-local-multi-node-cluster). diff --git a/docs/content/preview/tutorials/azure/azure-api-management.md b/docs/content/preview/tutorials/azure/azure-api-management.md index 9e8894bf1a8d..28edb1a14d86 100644 --- a/docs/content/preview/tutorials/azure/azure-api-management.md +++ b/docs/content/preview/tutorials/azure/azure-api-management.md @@ -37,20 +37,20 @@ Begin by deploying a multi-region, [geo-partitioned cluster](../../../yugabyte-c ![Geo-partitioned YugabyteDB deployment on Azure](/images/tutorials/azure/azure-private-link/yb-deployment.png "Geo-partitioned YugabyteDB deployment on Azure") -1. Enable public access on the cluster and add 0.0.0.0/0 to the cluster [IP Allow List](../../../yugabyte-cloud/cloud-secure-clusters/add-connections/). This setup allows connections to the cluster from all IP addresses. +1. Enable public access on the cluster and add 0.0.0.0/0 to the cluster [IP Allow List](/preview/yugabyte-cloud/cloud-secure-clusters/add-connections/). This setup allows connections to the cluster from all IP addresses. {{< note title="Note" >}} -In a production application, [Azure Private Link](../../../yugabyte-cloud/cloud-basics/cloud-vpcs/managed-endpoint-azure/) can be used with [private service endpoints](../../../yugabyte-cloud/cloud-basics/cloud-vpcs/managed-endpoint-azure/#create-a-pse-in-yugabytedb-managed) to create a secure connection between your application and database VPCs. +In a production application, [Azure Private Link](/preview/yugabyte-cloud/cloud-basics/cloud-vpcs/managed-endpoint-azure/) can be used with [private service endpoints](/preview/yugabyte-cloud/cloud-basics/cloud-vpcs/managed-endpoint-azure/#create-a-pse-in-yugabytedb-managed) to create a secure connection between your application and database VPCs. {{< /note >}} -1. Upon creation, save the credentials and [download the CA certificate](../../../tutorials/build-apps/cloud-add-ip/#download-your-cluster-certificate) once everything is up and running. This is essential for secure connections using the Node.js Smart Client. +1. Upon creation, save the credentials and [download the CA certificate](../../build-apps/cloud-add-ip/#download-your-cluster-certificate) once everything is up and running. This is essential for secure connections using the Node.js Smart Client. ## Create tables and insert records -Connect to your YugabyteDB cluster running on Azure via the [Cloud Shell](../../../yugabyte-cloud/cloud-connect/connect-cloud-shell/) and execute the following commands: +Connect to your YugabyteDB cluster running on Azure via the [Cloud Shell](/preview/yugabyte-cloud/cloud-connect/connect-cloud-shell/) and execute the following commands: 1. Create the _orders_ table and partition it by region. -1. Create partition tables using the automatically created [regional tablespaces](../../../yugabyte-cloud/cloud-basics/create-clusters/create-clusters-geopartition/#tablespaces). +1. Create partition tables using the automatically created [regional tablespaces](/preview/yugabyte-cloud/cloud-basics/create-clusters/create-clusters-geopartition/#tablespaces). 1. Seed the database with some orders. These records will be stored in the appropriate cluster node according to the supplied region. ## Develop an Azure function diff --git a/docs/content/preview/tutorials/azure/azure-private-link.md b/docs/content/preview/tutorials/azure/azure-private-link.md index 4fbe9ee44a15..06bea0ec277a 100644 --- a/docs/content/preview/tutorials/azure/azure-private-link.md +++ b/docs/content/preview/tutorials/azure/azure-private-link.md @@ -35,7 +35,7 @@ In the following sections, you will: ![3-node YugabyteDB deployment in uswest3](/images/tutorials/azure/azure-private-link/yb-deployment.png "3-node YugabyteDB deployment in uswest3") -Remember to save the credentials after creation and [download the CA certificate](../../../tutorials/build-apps/cloud-add-ip/#download-your-cluster-certificate) once operational, ensuring a secure connection through the Node.js Smart Client. +Remember to save the credentials after creation and [download the CA certificate](../../build-apps/cloud-add-ip/#download-your-cluster-certificate) once operational, ensuring a secure connection through the Node.js Smart Client. ## Get started with Azure diff --git a/docs/content/stable/drivers-orms/go/yb-pgx-reference.md b/docs/content/stable/drivers-orms/go/yb-pgx-reference.md index 6d95304540b1..b78b4f93d00a 100644 --- a/docs/content/stable/drivers-orms/go/yb-pgx-reference.md +++ b/docs/content/stable/drivers-orms/go/yb-pgx-reference.md @@ -236,13 +236,13 @@ For more details, see the [pgxpool package](https://pkg.go.dev/github.com/jackc/ To build a Go application that communicates securely over SSL with YugabyteDB database, you need the root certificate (`ca.crt`) of the YugabyteDB cluster. To generate these certificates and install them while launching the cluster, follow the instructions in [Create server certificates](../../../secure/tls-encryption/server-certificates/). -Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [CA certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [CA certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). For a YugabyteDB Aeon cluster, or a YugabyteDB cluster with SSL/TLS enabled, set the SSL-related environment variables as follows at the client side. ```sh $ export PGSSLMODE=verify-ca -$ export PGSSLROOTCERT=~/root.crt # Here, the CA certificate file is downloaded as `root.crt` under home directory. Modify your path accordingly. +$ export PGSSLROOTCERT=~/root.crt # CA certificate file is downloaded as `root.crt` under home directory. Modify your path accordingly. ``` | Environment Variable | Description | diff --git a/docs/content/stable/drivers-orms/java/postgres-jdbc.md b/docs/content/stable/drivers-orms/java/postgres-jdbc.md index d82a6f8bc424..800e6744336f 100644 --- a/docs/content/stable/drivers-orms/java/postgres-jdbc.md +++ b/docs/content/stable/drivers-orms/java/postgres-jdbc.md @@ -146,7 +146,7 @@ String yburl = "jdbc:postgresql://hostname:port/database?user=yugabyte&password= Connection conn = DriverManager.getConnection(yburl); ``` -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). ### Step 3: Write your application diff --git a/docs/content/stable/drivers-orms/java/yugabyte-jdbc.md b/docs/content/stable/drivers-orms/java/yugabyte-jdbc.md index 6714cb9d2b2c..2751e8649e2e 100644 --- a/docs/content/stable/drivers-orms/java/yugabyte-jdbc.md +++ b/docs/content/stable/drivers-orms/java/yugabyte-jdbc.md @@ -169,7 +169,7 @@ jdbc:yugabytedb://hostname:port/database?user=yugabyte&password=yugabyte&load-ba ssl=true&sslmode=verify-full&sslrootcert=~/.postgresql/root.crt ``` -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). To use load balancing and SSL mode verify-full with a cluster in YugabyteDB Aeon, you need to provide the additional `sslhostnameverifier` parameter, set to `com.yugabyte.ysql.YBManagedHostnameVerifier`. (Available in driver version 42.3.5-yb-2 or later. For previous versions of the driver, use `verify-ca`.) diff --git a/docs/content/stable/drivers-orms/nodejs/postgres-node-driver.md b/docs/content/stable/drivers-orms/nodejs/postgres-node-driver.md index 882123bc2178..c0bf091dbee6 100644 --- a/docs/content/stable/drivers-orms/nodejs/postgres-node-driver.md +++ b/docs/content/stable/drivers-orms/nodejs/postgres-node-driver.md @@ -115,7 +115,7 @@ const config = { } ``` -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). Refer to [Configure SSL/TLS](../postgres-pg-reference/#configure-ssl-tls) for more information on node-postgresql default and supported SSL modes, and other examples for setting up your connection strings when using SSL. diff --git a/docs/content/stable/drivers-orms/nodejs/postgres-pg-reference.md b/docs/content/stable/drivers-orms/nodejs/postgres-pg-reference.md index 6bc51af72cb5..5e26a9319af5 100644 --- a/docs/content/stable/drivers-orms/nodejs/postgres-pg-reference.md +++ b/docs/content/stable/drivers-orms/nodejs/postgres-pg-reference.md @@ -142,7 +142,7 @@ client To build a Node.js application that communicates securely over SSL, get the root certificate (`ca.crt`) of the YugabyteDB Cluster. If certificates are not generated yet, follow the instructions in [Create server certificates](../../../secure/tls-encryption/server-certificates/). -Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [Download your cluster certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +Because a YugabyteDB Aeon cluster is always configured with SSL/TLS, you don't have to generate any certificate but only set the client-side SSL configuration. To fetch your root certificate, refer to [Download your cluster certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). The node-postgres driver allows you to avoid including the parameters like `sslcert`, `sslkey`, `sslrootcert`, or `sslmode` in the connection string. You can pass the object which includes `connectionString` and `ssl` object which has various fields including the following: diff --git a/docs/content/stable/drivers-orms/nodejs/yugabyte-node-driver.md b/docs/content/stable/drivers-orms/nodejs/yugabyte-node-driver.md index ae630204082c..87e3d2a1e4a2 100644 --- a/docs/content/stable/drivers-orms/nodejs/yugabyte-node-driver.md +++ b/docs/content/stable/drivers-orms/nodejs/yugabyte-node-driver.md @@ -109,7 +109,7 @@ Refer to [Configure SSL/TLS](../postgres-pg-reference/#configure-ssl-tls) for mo #### Use SSL with YugabyteDB Aeon -If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). With clusters in YugabyteDB Aeon, you can't use SSL mode verify-full; other SSL modes are supported. To use the equivalent of verify-full, don't set the `sslmode` or `sslrootcert` parameters in your connection string; instead, use the `ssl` object with the following parameters: diff --git a/docs/content/stable/drivers-orms/orms/nodejs/ysql-prisma.md b/docs/content/stable/drivers-orms/orms/nodejs/ysql-prisma.md index e2ef5b9be434..d9bb5d4d0d1f 100644 --- a/docs/content/stable/drivers-orms/orms/nodejs/ysql-prisma.md +++ b/docs/content/stable/drivers-orms/orms/nodejs/ysql-prisma.md @@ -93,7 +93,7 @@ DATABASE_URL="postgresql://:@:/" If you have a YugabyteDB Aeon cluster, do the following: -1. Download your [cluster certificate](../../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +1. Download your [cluster certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). 1. Install OpenSSL, if not present. diff --git a/docs/content/stable/drivers-orms/python/postgres-psycopg2.md b/docs/content/stable/drivers-orms/python/postgres-psycopg2.md index 030d474b2e0b..bb53f953b49e 100644 --- a/docs/content/stable/drivers-orms/python/postgres-psycopg2.md +++ b/docs/content/stable/drivers-orms/python/postgres-psycopg2.md @@ -129,7 +129,7 @@ The following is an example for connecting to YugabyteDB with SSL encryption ena conn = psycopg2.connect("host= port=5433 dbname=yugabyte user= password= sslmode=verify-full sslrootcert=/Users/my-user/Downloads/root.crt") ``` -If you have created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +If you have created a cluster on YugabyteDB Aeon, use the cluster credentials and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). ### Step 3: Write your application diff --git a/docs/content/stable/drivers-orms/python/yugabyte-psycopg2.md b/docs/content/stable/drivers-orms/python/yugabyte-psycopg2.md index 09d6068a2109..34732234add6 100644 --- a/docs/content/stable/drivers-orms/python/yugabyte-psycopg2.md +++ b/docs/content/stable/drivers-orms/python/yugabyte-psycopg2.md @@ -184,7 +184,7 @@ The following is an example for connecting to a YugabyteDB cluster with SSL enab conn = psycopg2.connect("host= port=5433 dbname=yugabyte user= password= load_balance=true sslmode=verify-full sslrootcert=/path/to/root.crt") ``` -The Yugabyte Psycopg2 smart driver does not support SSL mode verify-full for clusters in YugabyteDB Aeon. Use verify-ca or the upstream psycopg2 driver. If your cluster is on YugabyteDB Aeon, use the cluster credentials for user and password, and [download the SSL Root certificate](../../../yugabyte-cloud/cloud-connect/connect-applications/). +The Yugabyte Psycopg2 smart driver does not support SSL mode verify-full for clusters in YugabyteDB Aeon. Use verify-ca or the upstream psycopg2 driver. If your cluster is on YugabyteDB Aeon, use the cluster credentials for user and password, and [download the SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). ### Step 3: Write your application diff --git a/docs/content/stable/drivers-orms/rust/rust-postgres-reference.md b/docs/content/stable/drivers-orms/rust/rust-postgres-reference.md index c0709579435e..1ee05a6277f3 100644 --- a/docs/content/stable/drivers-orms/rust/rust-postgres-reference.md +++ b/docs/content/stable/drivers-orms/rust/rust-postgres-reference.md @@ -353,7 +353,7 @@ The following is an example connection URL for connecting to a YugabyteDB cluste "postgresql://127.0.0.1:5434/yugabyte?user=yugabyte&password=yugabyte&load_balance=true&sslmode=require" ``` -If you created a cluster on [YugabyteDB Aeon](../../../yugabyte-cloud/), use the cluster credentials and download the [SSL Root certificate](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). +If you created a cluster on YugabyteDB Aeon, use the cluster credentials and download the [SSL Root certificate](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate). The following is an example application for connecting to a YugabyteDB cluster with SSL enabled: diff --git a/docs/content/stable/explore/change-data-capture/using-logical-replication/get-started.md b/docs/content/stable/explore/change-data-capture/using-logical-replication/get-started.md index b30bb0e0b128..9f8e20a1dc33 100644 --- a/docs/content/stable/explore/change-data-capture/using-logical-replication/get-started.md +++ b/docs/content/stable/explore/change-data-capture/using-logical-replication/get-started.md @@ -173,7 +173,7 @@ To start the services needed for this tutorial, you must: - [Start Zookeeper](#start-zookeeper) - [Start Kafka](#start-kafka) -- [Start a YugabyteDB database](#start-a-yugabytedb-database) +- [Start a YugabyteDB database](#start-yugabytedb) - [Start Kafka Connect](#start-kafka-connect) #### Start Zookeeper @@ -202,9 +202,11 @@ In this tutorial, you will always connect to Kafka from in a Docker container. A {{< /note >}} -#### Start a YugabyteDB database +#### Start YugabyteDB -At this point, you have started Zookeeper and Kafka, but you still need a database server from which Debezium can capture changes. In this procedure, you start a YugabyteDB instance with an example database. Follow the [Quick Start](../../../../quick-start) to start an instance using yugabyted. +At this point, you have started Zookeeper and Kafka, but you still need a database server from which Debezium can capture changes. In this procedure, you start a YugabyteDB instance with an example database. The example uses sample data in SQL scripts that are included with your YugabyteDB installation in the `share` directory. + +Follow the [Quick Start](../../../../quick-start) to start an instance using yugabyted. {{< note title="Note" >}} @@ -288,10 +290,10 @@ After starting YugabyteDB, you start the Kafka Connect service. This service exp 1. Open a new terminal, and use it to start the Kafka Connect service in a container. - The following command runs a new container using the `dz.2.5.2.yb.2024.1.SNAPSHOT.1` version of the `quay.io/yugabyte/ybdb-debezium` image: + The following command runs a new container using the `dz.2.5.2.yb.2024.1` version of the `quay.io/yugabyte/ybdb-debezium` image: ```sh - docker run -it --rm --name connect -p 8083:8083 -p 1976:1976 -e GROUP_ID=1 -e CONFIG_STORAGE_TOPIC=my_connect_configs -e OFFSET_STORAGE_TOPIC=my_connect_offsets -e STATUS_STORAGE_TOPIC=my_connect_statuses -e CLASSPATH=/kafka/connect/ --link zookeeper:zookeeper --link kafka:kafka quay.io/yugabyte/ybdb-debezium:dz.2.5.2.yb.2024.1.SNAPSHOT.1 + docker run -it --rm --name connect -p 8083:8083 -p 1976:1976 -e GROUP_ID=1 -e CONFIG_STORAGE_TOPIC=my_connect_configs -e OFFSET_STORAGE_TOPIC=my_connect_offsets -e STATUS_STORAGE_TOPIC=my_connect_statuses -e CLASSPATH=/kafka/connect/ --link zookeeper:zookeeper --link kafka:kafka quay.io/yugabyte/ybdb-debezium:dz.2.5.2.yb.2024.1 ``` 1. Verify that Kafka Connect started and is ready to accept connections. You should see output similar to the following: @@ -449,7 +451,7 @@ Using KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://172.17.0.7:9092 Using KAFKA_BROKER=172.17.0.3:9092 Contents of topic dbserver1.public.products: ... -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"before"},{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"name":"event.block","version":1,"field":"transaction"}],"optional":false,"name":"dbserver1.public.products.Envelope","version":1},"payload":{"before":null,"after":{"id":{"value":147,"set":true},"created_at":{"value":1500306107286000,"set":true},"category":{"value":"Doohickey","set":true},"ean":{"value":"6590063715","set":true},"price":{"value":44.4315141414441,"set":true},"quantity":{"value":5000,"set":true},"rating":{"value":4.6,"set":true},"title":{"value":"Mediocre Wool Toucan","set":true},"vendor":{"value":"Bradtke, Wilkinson and Reilly","set":true}},"source":{"version":"dz.2.5.2.yb.2024.1-SNAPSHOT","connector":"postgresql","name":"dbserver1","ts_ms":1721400304248,"snapshot":"true","db":"yugabyte","sequence":"[null,\"2\"]","schema":"public","table":"products","txId":2,"lsn":2,"xmin":null},"op":"r","ts_ms":1721400309609,"transaction":null}} +{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"before"},{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"default":0,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":false,"name":"id","field":"id"},{"type":"struct","fields":[{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"created_at","field":"created_at"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"category","field":"category"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"ean","field":"ean"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"price","field":"price"},{"type":"struct","fields":[{"type":"int32","optional":true,"default":5000,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"quantity","field":"quantity"},{"type":"struct","fields":[{"type":"double","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"rating","field":"rating"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"title","field":"title"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"value"},{"type":"boolean","optional":false,"field":"set"}],"optional":true,"name":"vendor","field":"vendor"}],"optional":true,"name":"dbserver1.public.products.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"name":"event.block","version":1,"field":"transaction"}],"optional":false,"name":"dbserver1.public.products.Envelope","version":1},"payload":{"before":null,"after":{"id":{"value":147,"set":true},"created_at":{"value":1500306107286000,"set":true},"category":{"value":"Doohickey","set":true},"ean":{"value":"6590063715","set":true},"price":{"value":44.4315141414441,"set":true},"quantity":{"value":5000,"set":true},"rating":{"value":4.6,"set":true},"title":{"value":"Mediocre Wool Toucan","set":true},"vendor":{"value":"Bradtke, Wilkinson and Reilly","set":true}},"source":{"version":"dz.2.5.2.yb.2024.1","connector":"postgresql","name":"dbserver1","ts_ms":1721400304248,"snapshot":"true","db":"yugabyte","sequence":"[null,\"2\"]","schema":"public","table":"products","txId":2,"lsn":2,"xmin":null},"op":"r","ts_ms":1721400309609,"transaction":null}} ... ``` diff --git a/docs/content/stable/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md b/docs/content/stable/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md index e5485909a2e3..3ce9c225fa24 100644 --- a/docs/content/stable/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md +++ b/docs/content/stable/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md @@ -1053,7 +1053,7 @@ If you have a YugabyteDB cluster with SSL enabled, you need to obtain the root c * [Local deployments](../../../../secure/tls-encryption/) * [YugabyteDB Anywhere](../../../../yugabyte-platform/security/enable-encryption-in-transit/#connect-to-a-ysql-endpoint-with-tls) -* [YugabyteDB Aeon](../../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate) +* [YugabyteDB Aeon](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate) {{< /note >}} diff --git a/jenkins_jobs.yml b/jenkins_jobs.yml index 3bf2e1571164..947a763dec56 100644 --- a/jenkins_jobs.yml +++ b/jenkins_jobs.yml @@ -29,21 +29,22 @@ jobs: architecture: aarch64 release_artifact: true - - os: mac12 + - os: mac13 compiler: clang build_type: release architecture: arm64 test_opts: YB_TEST_YB_CONTROLLER=0 release_artifact: true - - os: mac12 + - os: mac13 compiler: clang build_type: release architecture: x86_64 - test_opts: YB_TEST_YB_CONTROLLER=0 + test_opts: YB_COMPILE_ONLY=1 release_artifact: true - os: ubuntu22.04 compiler: clang17 build_type: debug release_artifact: false + test_opts: YB_COMPILE_ONLY=1 diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java index 1f46a6a85ff1..f612214f4001 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverse.java @@ -238,7 +238,32 @@ public void run() { .setSubTaskGroupType(SubTaskGroupType.ConfigureUniverse); } - createConfigureUniverseTasks(primaryCluster, null); + // Params for master_join_existing_universe gflag update + Runnable nonRestartMasterGflagUpgrade = null; + if (KubernetesUtil.isNonRestartGflagsUpgradeSupported( + primaryCluster.userIntent.ybSoftwareVersion)) { + KubernetesGflagsUpgradeCommonParams gflagsParams = + new KubernetesGflagsUpgradeCommonParams(universe, primaryCluster); + nonRestartMasterGflagUpgrade = + () -> + upgradePodsNonRestart( + universe.getName(), + // Use generated placement since gflagsParams placement will not have masters + // populated. + placement, + gflagsParams.getMasterAddresses(), + ServerType.MASTER, + gflagsParams.getYbSoftwareVersion(), + gflagsParams.getUniverseOverrides(), + gflagsParams.getAzOverrides(), + gflagsParams.isNewNamingStyle(), + false /* isReadOnlyCluster */, + gflagsParams.isEnableYbc(), + gflagsParams.getYbcSoftwareVersion()); + } + + createConfigureUniverseTasks( + primaryCluster, null /* masterNodes */, nonRestartMasterGflagUpgrade); // Run all the tasks. getRunnableTask().runSubTasks(); } catch (Throwable t) { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java index 9b4970b39fee..63f51bf19a19 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundle.java @@ -1,5 +1,7 @@ package com.yugabyte.yw.commissioner.tasks; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Throwables; import com.google.inject.Inject; import com.typesafe.config.Config; @@ -121,11 +123,11 @@ public Path generateBundle(SupportBundle supportBundle) // add the supportBundle metadata into the bundle try { + JsonNode sbJson = Json.toJson(supportBundle); + ((ObjectNode) sbJson).remove("status"); + ((ObjectNode) sbJson).remove("sizeInBytes"); supportBundleUtil.saveMetadata( - customer, - bundlePath.toAbsolutePath().toString(), - Json.toJson(supportBundle), - "manifest.json"); + customer, bundlePath.toAbsolutePath().toString(), sbJson, "manifest.json"); } catch (Exception e) { // Log the error and continue with the rest of support bundle collection. log.error("Error occurred while collecting support bundle manifest json", e); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateUniverse.java index 8a1818098c8b..e6d4ef7cb8a5 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateUniverse.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CreateUniverse.java @@ -204,7 +204,7 @@ public void run() { .setSubTaskGroupType(SubTaskGroupType.ConfigureUniverse); } - createConfigureUniverseTasks(primaryCluster, newMasters); + createConfigureUniverseTasks(primaryCluster, newMasters, null /* gflagsUpgradeSubtasks */); // Create Load Balancer map to add nodes to load balancer Map loadBalancerMap = diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java index 633d2281bde6..1361706e629c 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/KubernetesTaskBase.java @@ -56,6 +56,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import play.libs.Json; @Slf4j @@ -749,9 +750,9 @@ private Map getServersToUpdateAzMap( KubernetesPlacement placement, ServerType serverType) { Map serversToUpdate = new HashMap<>(); if (serverType.equals(ServerType.EITHER)) { - placement.masters.keySet().parallelStream() + placement.masters.keySet().stream() .forEach(azUUID -> serversToUpdate.put(azUUID, ServerType.MASTER)); - placement.tservers.keySet().parallelStream() + placement.tservers.keySet().stream() .forEach( azUUID -> { if (serversToUpdate.containsKey(azUUID)) { @@ -760,14 +761,12 @@ private Map getServersToUpdateAzMap( serversToUpdate.put(azUUID, ServerType.TSERVER); } }); + } else if (serverType.equals(ServerType.MASTER)) { + placement.masters.keySet().stream() + .forEach(azUUID -> serversToUpdate.put(azUUID, ServerType.MASTER)); } else { - if (serverType.equals(ServerType.MASTER)) { - placement.masters.keySet().parallelStream() - .forEach(azUUID -> serversToUpdate.put(azUUID, ServerType.MASTER)); - } else { - placement.tservers.keySet().parallelStream() - .forEach(azUUID -> serversToUpdate.put(azUUID, ServerType.TSERVER)); - } + placement.tservers.keySet().stream() + .forEach(azUUID -> serversToUpdate.put(azUUID, ServerType.TSERVER)); } return serversToUpdate; } @@ -1648,6 +1647,11 @@ public KubernetesCommandExecutor createKubernetesExecutorTaskForServerType( params.universeDetails = taskParams(); params.universeConfig = universe.getConfig(); } + // Case when new Universe is being created, we set the gflag "master_join_existing_cluster" + // to 'false'. + if ((commandType == CommandType.HELM_INSTALL) && StringUtils.isNotEmpty(masterAddresses)) { + params.masterJoinExistingCluster = false; + } params.masterPartition = masterPartition; params.tserverPartition = tserverPartition; params.enableNodeToNodeEncrypt = primaryCluster.userIntent.enableNodeToNodeEncrypt; diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java index 9a6a898f5f45..84ff3be96a84 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java @@ -997,15 +997,18 @@ public SubTaskGroup createGFlagsOverrideTasks( } public void createConfigureUniverseTasks( - Cluster primaryCluster, @Nullable Collection masterNodes) { + Cluster primaryCluster, + @Nullable Collection masterNodes, + @Nullable Runnable gflagsUpgradeSubtasks) { // Wait for a Master Leader to be elected. createWaitForMasterLeaderTask().setSubTaskGroupType(SubTaskGroupType.ConfigureUniverse); + // Update the gflags to set master_join_existing_universe to true. if (CollectionUtils.isNotEmpty(masterNodes) && primaryCluster.userIntent.providerType != CloudType.kubernetes) { - // Update the gflags to set master_join_existing_universe to false. - // It is not set for k8s universe because this restarts the pods. createGFlagsOverrideTasks(masterNodes, ServerType.MASTER, null /* param customizer */); + } else if (gflagsUpgradeSubtasks != null) { + gflagsUpgradeSubtasks.run(); } // Persist the placement info into the YB master leader. diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateLoadBalancerConfig.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateLoadBalancerConfig.java index cf4ce633fc07..677aae330a0d 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateLoadBalancerConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UpdateLoadBalancerConfig.java @@ -1,6 +1,7 @@ package com.yugabyte.yw.commissioner.tasks; import com.yugabyte.yw.commissioner.BaseTaskDependencies; +import com.yugabyte.yw.commissioner.UserTaskDetails.SubTaskGroupType; import com.yugabyte.yw.common.PlacementInfoUtil; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.models.AvailabilityZone; @@ -38,9 +39,9 @@ public void run() { log.info("Started {} task for univ uuid={}", getName(), taskParams().getUniverseUUID()); Universe universe = getUniverse(); try { - // Update the universe DB with the changes to be performed and set the 'updateInProgress' flag - // to prevent other updates from happening. - universe = lockAndFreezeUniverseForUpdate(taskParams().expectedUniverseVersion, null); + // Lock the universe but don't freeze it because this task doesn't perform critical updates to + // universe metadata. + universe = lockUniverse(-1 /* expectedUniverseVersion */); // Get expected LB config Map newLbMap = @@ -62,6 +63,9 @@ public void run() { }; saveUniverseDetails(updater); + createMarkUniverseUpdateSuccessTasks() + .setSubTaskGroupType(SubTaskGroupType.ConfigureUniverse); + getRunnableTask().runSubTasks(); } catch (Exception e) { log.error("Task Errored out with: " + e); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java index 0373b9f3cc75..17e4a145b911 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java @@ -243,6 +243,8 @@ public static class Params extends UniverseTaskParams { public boolean usePreviousGflagsChecksum = false; public boolean createNamespacedService = false; public Set deleteServiceNames; + // Only set false for create universe case initially + public boolean masterJoinExistingCluster = true; } protected KubernetesCommandExecutor.Params taskParams() { @@ -1138,6 +1140,9 @@ private String generateHelmOverride() { XClusterConfigTaskBase.XCLUSTER_ROOT_CERTS_DIR_GFLAG, taskUniverseDetails.xClusterInfo.sourceRootCertDirPath); } + if (taskParams().masterJoinExistingCluster) { + masterGFlags.put(GFlagsUtil.MASTER_JOIN_EXISTING_UNIVERSE, "true"); + } if (!masterGFlags.isEmpty()) { gflagOverrides.put("master", masterGFlags); } diff --git a/managed/src/main/java/com/yugabyte/yw/common/KubernetesUtil.java b/managed/src/main/java/com/yugabyte/yw/common/KubernetesUtil.java index d2d8f1dff05d..ec55f3d2e0fc 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/KubernetesUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/KubernetesUtil.java @@ -73,7 +73,7 @@ public static boolean isNonRestartGflagsUpgradeSupported(String universeSoftware MIN_VERSION_NON_RESTART_GFLAGS_UPGRADE_SUPPORT_STABLE, MIN_VERSION_NON_RESTART_GFLAGS_UPGRADE_SUPPORT_PREVIEW, true) - > 0; + >= 0; } public static boolean isNamespacedServiceSupported(String universeSoftwareVersion) { @@ -82,7 +82,7 @@ public static boolean isNamespacedServiceSupported(String universeSoftwareVersio MIN_VERSION_NAMESPACED_SERVICE_SUPPORT_STABLE, MIN_VERSION_NAMESPACED_SERVICE_SUPPORT_PREVIEW, true) - > 0; + >= 0; } // ToDo: Old k8s provider needs to be fixed, so that we can get diff --git a/managed/src/main/resources/alert/alert_templates.yml b/managed/src/main/resources/alert/alert_templates.yml index 0f5dac96287d..e5881278b9c5 100644 --- a/managed/src/main/resources/alert/alert_templates.yml +++ b/managed/src/main/resources/alert/alert_templates.yml @@ -1453,7 +1453,7 @@ templates: server_type="yb_ysqlserver",service_type="SQLProcessor",service_method=~"SelectStmt|InsertStmt|UpdateStmt|DeleteStmt|Transactions"}[5m])) / sum by (universe_uuid, service_method)(rate(rpc_latency_count{universe_uuid="__universeUuid__",export_type="ysql_export",server_type="yb_ysqlserver", service_type="SQLProcessor",service_method=~"SelectStmt|InsertStmt|UpdateStmt|DeleteStmt|Transactions"}[5m]))) - {{ query_condition }} {{ query_threshold }} + / 1000 {{ query_condition }} {{ query_threshold }} defaultThresholdMap: SEVERE: paramName: yb.alert.max_ysql_opavg_latency @@ -1499,7 +1499,7 @@ templates: service_type="SQLProcessor",service_method=~"SelectStmt|InsertStmt|UpdateStmt|DeleteStmt|Transaction"}[5m])) / sum by (service_method)(rate(rpc_latency_count{universe_uuid="__universeUuid__",export_type="cql_export",server_type="yb_cqlserver", service_type="SQLProcessor",service_method=~"SelectStmt|InsertStmt|UpdateStmt|DeleteStmt|Transaction"}[5m]))) - {{ query_condition }} {{ query_threshold }} + / 1000 {{ query_condition }} {{ query_threshold }} defaultThresholdMap: SEVERE: paramName: yb.alert.max_ycql_opavg_latency @@ -1521,7 +1521,7 @@ templates: description: P99 latency of YCQL operations is above threshold queryTemplate: max by (universe_uuid)(rpc_latency{universe_uuid="__universeUuid__",server_type="yb_cqlserver", service_type="SQLProcessor",service_method=~"SelectStmt|InsertStmt|UpdateStmt|DeleteStmt|OtherStmts|Transaction",quantile="p99"}) - {{ query_condition }} {{ query_threshold }} + /1000 {{ query_condition }} {{ query_threshold }} defaultThresholdMap: SEVERE: paramName: yb.alert.max_ycql_p99_latency @@ -1533,7 +1533,7 @@ templates: affected_node_names: >- {{ range $index, $element := query "max by (universe_uuid, node_name)(rpc_latency{universe_uuid='{{ $labels.universe_uuid }}',server_type='yb_cqlserver', service_type='SQLProcessor',service_method=~'SelectStmt|InsertStmt|UpdateStmt|DeleteStmt|OtherStmts|Transaction',quantile='p99'}) - {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} + / 1000 {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} annotations: summary: >- YCQL P99 latency for universe '{{ $labels.source_name }}' is @@ -1690,7 +1690,7 @@ templates: description: Average read latency of tablet server is above threshold queryTemplate: (avg by (universe_uuid) (rate(rpc_latency_sum{export_type="tserver_export", service_type="TabletServerService", universe_uuid="__universeUuid__", server_type="yb_tserver", service_method="Read"}[5m]))) / (avg by (universe_uuid) (rate(rpc_latency_count{export_type="tserver_export", service_type="TabletServerService", universe_uuid="__universeUuid__", server_type="yb_tserver", service_method="Read"}[5m])) * 1000) - {{ query_condition }} {{ query_threshold }} + / 1000 {{ query_condition }} {{ query_threshold }} createForNewCustomer: false defaultThresholdMap: SEVERE: @@ -1704,7 +1704,7 @@ templates: {{ range $index, $element := query " (avg by (universe_uuid, node_name) (rate(rpc_latency_sum{export_type='tserver_export', service_type='TabletServerService', universe_uuid='{{ $labels.universe_uuid }}', server_type='yb_tserver', service_method='Read'}[5m]))) / (avg by (universe_uuid, node_name) (rate(rpc_latency_count{export_type='tserver_export', service_type='TabletServerService', universe_uuid='{{ $labels.universe_uuid }}', server_type='yb_tserver', service_method='Read'}[5m])) * 1000) - {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} + / 1000 {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} annotations: summary: >- Average read latency of tablet server for universe '{{ $labels.source_name }}' @@ -1718,7 +1718,7 @@ templates: description: Average write latency of tablet server is above threshold queryTemplate: (avg by (universe_uuid) (rate(rpc_latency_sum{export_type="tserver_export", service_type="TabletServerService", universe_uuid="__universeUuid__", server_type="yb_tserver", service_method="Write"}[5m]))) / (avg by (universe_uuid) (rate(rpc_latency_count{export_type="tserver_export", service_type="TabletServerService", universe_uuid="__universeUuid__", server_type="yb_tserver", service_method="Write"}[5m])) * 1000) - {{ query_condition }} {{ query_threshold }} + / 1000 {{ query_condition }} {{ query_threshold }} createForNewCustomer: false defaultThresholdMap: SEVERE: @@ -1732,7 +1732,7 @@ templates: {{ range $index, $element := query " (avg by (universe_uuid, node_name) (rate(rpc_latency_sum{export_type='tserver_export', service_type='TabletServerService', universe_uuid='{{ $labels.universe_uuid }}', server_type='yb_tserver', service_method='Write'}[5m]))) / (avg by (universe_uuid, node_name) (rate(rpc_latency_count{export_type='tserver_export', service_type='TabletServerService', universe_uuid='{{ $labels.universe_uuid }}', server_type='yb_tserver', service_method='Write'}[5m])) * 1000) - {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} + / 1000 {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} annotations: summary: >- Average write latency of tablet server for universe '{{ $labels.source_name }}' @@ -1921,7 +1921,7 @@ templates: affected_node_names: >- {{ range $index, $element := query "max by (universe_uuid, node_name) (yb_node_clock_drift_check_ms{universe_uuid='{{ $labels.universe_uuid }}'}) - / 1000 {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} + {{ query_condition }} {{ query_threshold }}" }}{{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }} annotations: summary: >- Clock drift is high for universe '{{ $labels.source_name }}'. diff --git a/managed/src/main/resources/db/migration/default_/common/V378__Update_ClockDrift_Alert.sql b/managed/src/main/resources/db/migration/default_/common/V378__Update_ClockDrift_Alert.sql new file mode 100644 index 000000000000..545a5b4e2529 --- /dev/null +++ b/managed/src/main/resources/db/migration/default_/common/V378__Update_ClockDrift_Alert.sql @@ -0,0 +1,2 @@ +update alert_definition set config_written = false where configuration_uuid IN + (select uuid from alert_configuration where template = 'NODE_CLOCK_DRIFT') \ No newline at end of file diff --git a/managed/src/main/resources/reference.conf b/managed/src/main/resources/reference.conf index 7268c913df41..75c506e148f0 100644 --- a/managed/src/main/resources/reference.conf +++ b/managed/src/main/resources/reference.conf @@ -477,8 +477,9 @@ yb { } authtoken { - # Expiry time of auth token in days - token_expiry = 7 days + # Expiry time of auth token + # https://github.com/lightbend/config/blob/main/HOCON.md#duration-format for supported formats + token_expiry = 12 hours } # We delete completed task info form database. diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java index 6113ef4e5ac0..e9590866f14d 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateKubernetesUniverseTest.java @@ -6,6 +6,7 @@ import static com.yugabyte.yw.commissioner.tasks.subtasks.KubernetesCommandExecutor.CommandType.APPLY_SECRET; import static com.yugabyte.yw.commissioner.tasks.subtasks.KubernetesCommandExecutor.CommandType.CREATE_NAMESPACE; import static com.yugabyte.yw.commissioner.tasks.subtasks.KubernetesCommandExecutor.CommandType.HELM_INSTALL; +import static com.yugabyte.yw.commissioner.tasks.subtasks.KubernetesCommandExecutor.CommandType.HELM_UPGRADE; import static com.yugabyte.yw.commissioner.tasks.subtasks.KubernetesCommandExecutor.CommandType.POD_INFO; import static com.yugabyte.yw.common.ApiUtils.getTestUserIntent; import static com.yugabyte.yw.common.AssertHelper.assertJsonEqual; @@ -363,6 +364,8 @@ private void setupCommon() { TaskType.InstallingThirdPartySoftware, TaskType.WaitForServer, TaskType.WaitForMasterLeader, + TaskType.KubernetesCommandExecutor, + TaskType.WaitForDuration, TaskType.UpdatePlacementInfo, TaskType.WaitForTServerHeartBeats, TaskType.SwamperTargetsFileUpdate, @@ -387,6 +390,8 @@ private List getExpectedCreateUniverseTaskResults() { Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), + Json.toJson(ImmutableMap.of("commandType", HELM_UPGRADE.name())), + Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("removeFile", false)), @@ -406,7 +411,25 @@ private List getTaskPositionsToSkip(boolean skipNamespace) { private List getTaskCountPerPosition(int namespaceTasks, int parallelTasks) { return ImmutableList.of( - 1, namespaceTasks, parallelTasks, parallelTasks, 0, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1); + 1, + namespaceTasks, + parallelTasks, + parallelTasks, + 0, + 1, + 1, + 3, + 1, + parallelTasks, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1); } private void assertTaskSequence( diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutorTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutorTest.java index 1f94f97e80ae..aca147fab54d 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutorTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutorTest.java @@ -310,13 +310,6 @@ private Map getExpectedOverrides(boolean exposeAll) { expectedOverrides.put("ip_version_support", "v6_only"); } - // For all tests but 1, value should default to true. - if (defaultUserIntent.enableExposingService == ExposingServiceState.UNEXPOSED) { - expectedOverrides.put("enableLoadBalancer", false); - } else { - expectedOverrides.put("enableLoadBalancer", true); - } - Map partition = new HashMap<>(); partition.put("tserver", 0); partition.put("master", 0); @@ -331,6 +324,7 @@ private Map getExpectedOverrides(boolean exposeAll) { masterGFlags.put("placement_zone", defaultAZ.getCode()); masterGFlags.put( "placement_uuid", defaultUniverse.getUniverseDetails().getPrimaryCluster().uuid.toString()); + masterGFlags.put("master_join_existing_universe", "true"); gflagOverrides.put("master", masterGFlags); // Tserver flags. @@ -345,10 +339,6 @@ private Map getExpectedOverrides(boolean exposeAll) { // Put all the flags together. expectedOverrides.put("gflags", gflagOverrides); - Map ybcOverrides = new HashMap<>(); - ybcOverrides.put("enabled", false); - expectedOverrides.put("ybc", ybcOverrides); - Map universeConfig = defaultUniverse.getConfig(); if (universeConfig.getOrDefault(Universe.LABEL_K8S_RESOURCES, "false").equals("true")) { expectedOverrides.put( @@ -382,6 +372,14 @@ private Map getExpectedOverrides(boolean exposeAll) { } } expectedOverrides.put("disableYsql", !defaultUserIntent.enableYSQL); + + // For all tests but 1, value should default to true. + if (defaultUserIntent.enableExposingService == ExposingServiceState.UNEXPOSED) { + expectedOverrides.put("enableLoadBalancer", false); + } else { + expectedOverrides.put("enableLoadBalancer", true); + } + boolean helmLegacy = Universe.HelmLegacy.valueOf(universeConfig.get(Universe.HELM2_LEGACY)) == Universe.HelmLegacy.V2TO3; @@ -400,6 +398,10 @@ private Map getExpectedOverrides(boolean exposeAll) { yugabytedUiInfo.put("metricsSnapshotter", metricsSnapshotterInfo); expectedOverrides.put("yugabytedUi", yugabytedUiInfo); + Map ybcOverrides = new HashMap<>(); + ybcOverrides.put("enabled", false); + expectedOverrides.put("ybc", ybcOverrides); + expectedOverrides.put("defaultServiceScope", "AZ"); return expectedOverrides; } diff --git a/python/yugabyte/yb_dist_tests.py b/python/yugabyte/yb_dist_tests.py index d143c6c6a85c..d2e31b41b65e 100644 --- a/python/yugabyte/yb_dist_tests.py +++ b/python/yugabyte/yb_dist_tests.py @@ -216,6 +216,7 @@ def set_global_conf_from_dict(global_conf_dict: Dict[str, str]) -> GlobalTestCon 'www', 'yb_build.sh', 'build/venv', + 'build/venv-arm64', 'build/ybc', 'requirements.txt', 'requirements_frozen.txt', diff --git a/src/yb/integration-tests/tablet_health_manager-itest.cc b/src/yb/integration-tests/tablet_health_manager-itest.cc index dbb2a14b6d10..a4a8c569119e 100644 --- a/src/yb/integration-tests/tablet_health_manager-itest.cc +++ b/src/yb/integration-tests/tablet_health_manager-itest.cc @@ -12,6 +12,7 @@ // #include +#include #include #include @@ -29,6 +30,7 @@ #include "yb/tools/yb-admin_client.h" #include "yb/util/backoff_waiter.h" +#include "yb/util/curl_util.h" #include "yb/util/monotime.h" #include "yb/util/test_macros.h" #include "yb/util/tsan_util.h" @@ -36,6 +38,7 @@ DECLARE_int32(are_nodes_safe_to_take_down_timeout_buffer_ms); DECLARE_double(leader_failure_max_missed_heartbeat_periods); DECLARE_int32(raft_heartbeat_interval_ms); +DECLARE_int32(catalog_manager_bg_task_wait_ms); using std::string; using std::vector; @@ -43,6 +46,19 @@ using std::vector; namespace yb { namespace integration_tests { +template +Result parse(const std::string& s); + +template <> +Result parse(const std::string& s) { + std::size_t pos; + auto out = std::stoull(s, &pos); + if (pos != s.size()) { + return STATUS(InvalidArgument, "Couldn't parse string to unsigned integer: $0", s); + } + return out; +} + class AreNodesSafeToTakeDownItest : public YBTableTestBase { protected: void BeforeCreateTable() override { @@ -90,6 +106,32 @@ class AreNodesSafeToTakeDownItest : public YBTableTestBase { } const int kFollowerLagBoundMs = 1000 * kTimeMultiplier; + + template + Result GetLeaderMasterMetric(const std::string& metric_name) { + EasyCurl curl; + auto addr = external_mini_cluster_->GetLeaderMaster()->bound_http_hostport(); + faststring out; + RETURN_NOT_OK( + curl.FetchURL(Format("http://$0:$1/prometheus-metrics", addr.host(), addr.port()), &out)); + std::stringstream ss(out.ToString()); + for (std::string line; std::getline(ss, line, '\n');) { + if (!line.starts_with(metric_name)) { + continue; + } + std::vector words; + std::stringstream line_ss(line); + for (std::string word; std::getline(line_ss, word, ' ');) { + words.push_back(std::move(word)); + } + if (words.size() < 2) { + return STATUS_FORMAT(IllegalState, "Bad prometheus metric line: $0", line); + } + return parse(words[words.size() - 2]); + } + return STATUS_FORMAT( + NotFound, "Couldn't find metric $0 on prometheus-metrics page", metric_name); + } }; TEST_F(AreNodesSafeToTakeDownItest, HealthyCluster) { @@ -257,6 +299,9 @@ TEST_F(AreNodesSafeToTakeDownItest, GetFollowerUpdateDelay) { for (const auto& heartbeat_delay : resp.heartbeat_delay()) { ASSERT_LE(heartbeat_delay.last_heartbeat_delta_ms(), max_expected_heartbeat_time); } + auto max_actual_heartbeat_delay = + ASSERT_RESULT(GetLeaderMasterMetric("max_follower_heartbeat_delay")); + ASSERT_LE(max_actual_heartbeat_delay, max_expected_heartbeat_time); } // Stop one of the nodes and ensure its delay increases. @@ -271,7 +316,7 @@ TEST_F(AreNodesSafeToTakeDownItest, GetFollowerUpdateDelayWithStoppedNode) { ASSERT_NE(master_it, masters.end()); auto other_master = *master_it; other_master->Shutdown(); - auto sleep_time = FLAGS_raft_heartbeat_interval_ms * 3; + auto sleep_time = static_cast(FLAGS_raft_heartbeat_interval_ms * 3); SleepFor(MonoDelta::FromMilliseconds(sleep_time)); master::GetMasterHeartbeatDelaysRequestPB req; master::GetMasterHeartbeatDelaysResponsePB resp; @@ -286,7 +331,23 @@ TEST_F(AreNodesSafeToTakeDownItest, GetFollowerUpdateDelayWithStoppedNode) { ASSERT_GE(heartbeat_delay.last_heartbeat_delta_ms(), sleep_time); } } - + // Verify the heartbeat delay shows up in the metric max_follower_heartbeat_delay. This metric is + // updated by the catalog manager background task. The background task may not have run recently + // enough to pick up the delay so we retry a few times. + std::string failure_message; + ASSERT_OK(WaitFor( + [this, sleep_time, &failure_message]() -> Result { + auto max_actual_heartbeat_delay = + VERIFY_RESULT(GetLeaderMasterMetric("max_follower_heartbeat_delay")); + auto done = max_actual_heartbeat_delay >= sleep_time; + if (!done) { + failure_message = Format( + "Heartbeat delay from metric is $0, expecting it to be at least $1", + max_actual_heartbeat_delay, sleep_time); + } + return done; + }, + MonoDelta::FromMilliseconds(FLAGS_catalog_manager_bg_task_wait_ms * 3), failure_message)); ASSERT_OK(other_master->Restart()); } diff --git a/src/yb/master/catalog_manager.cc b/src/yb/master/catalog_manager.cc index 47f752ec94e3..31a34dd34769 100644 --- a/src/yb/master/catalog_manager.cc +++ b/src/yb/master/catalog_manager.cc @@ -527,6 +527,15 @@ METRIC_DEFINE_gauge_uint32(cluster, num_tablet_servers_dead, "heartbeat in the time interval defined by the gflag " "FLAGS_tserver_unresponsive_timeout_ms."); +METRIC_DEFINE_gauge_uint64(cluster, max_follower_heartbeat_delay, "The maximum heartbeat delay " + "among all master followers", + yb::MetricUnit::kMilliseconds, + "The number of milliseconds since the master leader has ACK'd a " + "heartbeat from all peers of the system catalog tablet. The master " + "leader does not ACK heartbeats from peers that have fallen behind due " + "to WAL GC so such peers will cause this metric to increase " + "indefinitely."); + METRIC_DEFINE_counter(cluster, create_table_too_many_tablets, "How many CreateTable requests have failed due to too many tablets", yb::MetricUnit::kRequests, "The number of CreateTable request errors due to attempting to create too many tablets."); @@ -976,6 +985,9 @@ Status CatalogManager::Init() { metric_create_table_too_many_tablets_ = METRIC_create_table_too_many_tablets.Instantiate(master_->metric_entity_cluster()); + metric_max_follower_heartbeat_delay_ = + METRIC_max_follower_heartbeat_delay.Instantiate(master_->metric_entity_cluster(), 0); + cdc_state_table_ = std::make_unique(master_->cdc_state_client_future()); RETURN_NOT_OK(xcluster_manager_->Init()); @@ -11679,6 +11691,22 @@ void CatalogManager::ReportMetrics() { auto num_servers = master_->ts_manager()->NumDescriptors(); metric_num_tablet_servers_dead_->set_value( narrow_cast(num_servers - num_live_servers)); + + // Report the max master follower heartbeat delay. + auto consensus_result = tablet_peer()->GetConsensus(); + if (consensus_result) { + MonoTime earliest_time = MonoTime::kUninitialized; + for (const auto& last_communication_time : + (*consensus_result)->GetFollowerCommunicationTimes()) { + earliest_time.MakeAtMost(last_communication_time.last_successful_communication); + } + int64_t time_in_ms = + earliest_time ? MonoTime::Now().GetDeltaSince(earliest_time).ToMilliseconds() : 0; + if (time_in_ms < 0) { + time_in_ms = 0; + } + metric_max_follower_heartbeat_delay_->set_value(static_cast(time_in_ms)); + } } void CatalogManager::ResetMetrics() { diff --git a/src/yb/master/catalog_manager.h b/src/yb/master/catalog_manager.h index 9469cc47cca4..27a33299d155 100644 --- a/src/yb/master/catalog_manager.h +++ b/src/yb/master/catalog_manager.h @@ -2379,6 +2379,8 @@ class CatalogManager : public tserver::TabletPeerLookupIf, scoped_refptr metric_create_table_too_many_tablets_; + scoped_refptr> metric_max_follower_heartbeat_delay_; + friend class ClusterLoadBalancer; // Policy for load balancing tablets on tablet servers. diff --git a/src/yb/tablet/tablet_peer.cc b/src/yb/tablet/tablet_peer.cc index 103b638419c7..f98553d067e6 100644 --- a/src/yb/tablet/tablet_peer.cc +++ b/src/yb/tablet/tablet_peer.cc @@ -943,7 +943,12 @@ void TabletPeer::GetInFlightOperations(Operation::TraceType trace_type, } Result TabletPeer::MaxPersistentOpId() const { - auto flush_op_ids = VERIFY_RESULT(tablet_->MaxPersistentOpId()); + auto tablet = shared_tablet(); + if (!tablet) { + // Tablet peer not yet initialized -- we could be doing tablet bootstrap still. + return OpId::Min(); + } + auto flush_op_ids = VERIFY_RESULT(tablet->MaxPersistentOpId()); return OpId::MinValid(flush_op_ids.intents, flush_op_ids.regular); } diff --git a/src/yb/util/monotime.cc b/src/yb/util/monotime.cc index c8cc1876267d..d083d8bf06e6 100644 --- a/src/yb/util/monotime.cc +++ b/src/yb/util/monotime.cc @@ -380,6 +380,12 @@ void MonoTime::MakeAtLeast(MonoTime rhs) { } } +void MonoTime::MakeAtMost(MonoTime rhs) { + if (rhs.Initialized() && (!Initialized() || value_ > rhs.value_)) { + value_ = rhs.value_; + } +} + // ------------------------------------------------------------------------------------------------ std::string FormatForComparisonFailureMessage(const MonoDelta& op, const MonoDelta& other) { diff --git a/src/yb/util/monotime.h b/src/yb/util/monotime.h index bc963536b262..1fdb52ba1aab 100644 --- a/src/yb/util/monotime.h +++ b/src/yb/util/monotime.h @@ -235,6 +235,8 @@ class MonoTime { // Set this time to the given value if it is lower than that or uninitialized. void MakeAtLeast(MonoTime rhs); + void MakeAtMost(MonoTime rhs); + TimePoint ToSteadyTimePoint() const { return value_; } diff --git a/yb_release_manifest.json b/yb_release_manifest.json index 6eb621566607..f17b315e8bee 100644 --- a/yb_release_manifest.json +++ b/yb_release_manifest.json @@ -5,6 +5,7 @@ }, ".": [ "$BUILD_ROOT/auto_flags.json", + "$BUILD_ROOT/gflag_allowlist.txt", "$BUILD_ROOT/master_flags.xml", "$BUILD_ROOT/tserver_flags.xml", "$BUILD_ROOT/version_metadata.json" diff --git a/yugabyted-ui/apiserver/cmd/server/handlers/api_cluster.go b/yugabyted-ui/apiserver/cmd/server/handlers/api_cluster.go index af1171bab6ef..a686a89139cc 100644 --- a/yugabyted-ui/apiserver/cmd/server/handlers/api_cluster.go +++ b/yugabyted-ui/apiserver/cmd/server/handlers/api_cluster.go @@ -13,10 +13,12 @@ import ( "github.com/labstack/echo/v4" ) - +const MOST_RECENT_TIMESTAMP string = "select ts " + + "from %s where metric='%s' and node='%s' limit 1;" const QUERY_LIMIT_ONE string = "select ts, value, details " + "from %s where metric='%s' and node='%s' limit 1;" - +const MRT_QUERY_METRICS string = "select ts, value, details " + + "from %s where metric='%s' and node='%s' and ts>=%d" // GetCluster - Get a cluster func (c *Container) GetCluster(ctx echo.Context) error { // Perform all necessary http requests asynchronously @@ -181,6 +183,7 @@ func (c *Container) GetCluster(ctx echo.Context) error { // and each tserver has the flag: // --use_client_to_server_encryption=true // If any flag on any server does not match, we don't say encryption in transit is enabled. + tServerSnapshotMap := make(map[string]string) isEncryptionInTransitEnabled := true for host, gFlagsTserverFuture := range gFlagsTserverFutures { tserverFlags := <-gFlagsTserverFuture @@ -202,6 +205,7 @@ func (c *Container) GetCluster(ctx echo.Context) error { } break } + tServerSnapshotMap[host] = tserverFlags.GFlags["metrics_snapshotter_interval_ms"] } // Use this to track gflag results that have been read, to avoid reading a future twice // which will hang @@ -312,30 +316,48 @@ func (c *Container) GetCluster(ctx echo.Context) error { // Thus, we will only count the metrics from non 127 addresses, unless all addresses // are 127, in which case we count only one of them. for _, host := range reducedNodeList { + var recentTs int64 + var snapshotterInterval int64 + var errDataType error + var refinedTs int64 + snapshotterIntervalStr := tServerSnapshotMap[host] + if snapshotterIntervalStr == "" { + snapshotterIntervalStr = "11000" //default value is set to 11000ms + } + snapshotterInterval, errDataType = strconv.ParseInt(snapshotterIntervalStr, 10, 64) + if errDataType != nil { + fmt.Println("Error: metrics_snapshotter_interval_ms not in proper type") + } + timestamp := fmt.Sprintf( + MOST_RECENT_TIMESTAMP, "system.metrics", "total_disk", hostToUuid[host]) + iter := session.Query(timestamp).Iter() + iter.Scan(&recentTs) + refinedTs = recentTs - snapshotterInterval / 2 query := fmt.Sprintf( - QUERY_LIMIT_ONE, "system.metrics", "total_disk", hostToUuid[host]) - iter := session.Query(query).Iter() + MRT_QUERY_METRICS, "system.metrics", "total_disk", hostToUuid[host], refinedTs) + iter = session.Query(query).Iter() var ts int64 var value int var details string - iter.Scan(&ts, &value, &details) + for iter.Scan(&ts, &value, &details) { + totalDiskGb += float64(value) / helpers.BYTES_IN_GB + } if err := iter.Close(); err != nil { c.logger.Errorf("error fetching total_disk data from %s: %s", host, err.Error()) continue - } else { - totalDiskGb += float64(value) / helpers.BYTES_IN_GB } query = fmt.Sprintf( - QUERY_LIMIT_ONE, "system.metrics", "free_disk", hostToUuid[host]) + MRT_QUERY_METRICS, "system.metrics", "free_disk", hostToUuid[host], refinedTs) + refinedTs = snapshotterInterval / 2 iter = session.Query(query).Iter() - iter.Scan(&ts, &value, &details) + for iter.Scan(&ts, &value, &details) { + freeDiskGb += float64(value) / helpers.BYTES_IN_GB + } if err := iter.Close(); err != nil { c.logger.Errorf("error fetching free_disk data from %s: %s", host, err.Error()) continue - } else { - freeDiskGb += float64(value) / helpers.BYTES_IN_GB } } // Get software version