Skip to content

Commit

Permalink
Merge pull request #29365 from teskje/mz_cluster_replica_status_history
Browse files Browse the repository at this point in the history 
adapter,controller: add replica status history
  • Loading branch information
@teskje
teskje authored Sep 12, 2024
2 parents dc80757 + c96b861 commit 95aa493
Show file tree
Hide file tree
Showing 21 changed files with 257 additions and 105 deletions.
23 changes: 19 additions & 4 deletions doc/user/content/sql/system-catalog/mz_internal.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ At this time, we do not make any guarantees about the exactness or freshness of
| Field | Type | Meaning |
| ------------------- | ------------ | -------- |
| `replica_id` | [`text`] | The ID of a cluster replica. |
| `process_id` | [`uint8`] | An identifier of a compute process within a replica. |
| `process_id` | [`uint8`] | The ID of a process within the replica. |
| `cpu_nano_cores` | [`uint8`] | Approximate CPU usage, in billionths of a vCPU core. |
| `memory_bytes` | [`uint8`] | Approximate RAM usage, in bytes. |
| `disk_bytes` | [`uint8`] | Approximate disk usage in bytes. |
Expand All @@ -193,7 +193,7 @@ At this time, we do not make any guarantees about the exactness or freshness of
| Field | Type | Meaning
| ---------------- | --------- | --------
| `replica_id` | [`text`] | The ID of a cluster replica.
| `process_id` | [`uint8`] | An identifier of a process within the replica.
| `process_id` | [`uint8`] | The ID of a process within the replica.
| `cpu_nano_cores` | [`uint8`] | Approximate CPU usage in billionths of a vCPU core.
| `memory_bytes` | [`uint8`] | Approximate memory usage in bytes.
| `disk_bytes` | [`uint8`] | Approximate disk usage in bytes.
Expand All @@ -213,6 +213,21 @@ of each process in each cluster replica in the system.
| `reason` | [`text`] | If the cluster replica is in a `offline` state, the reason (if available). For example, `oom-killed`. |
| `updated_at` | [`timestamp with time zone`] | The time at which the status was last updated. |

## `mz_cluster_replica_status_history`

{{< warn-if-unreleased v0.116 >}}
The `mz_cluster_replica_status_history` table records status changes
for all processes of all extant cluster replicas.

<!-- RELATION_SPEC mz_internal.mz_cluster_replica_status_history -->
| Field | Type | Meaning
|---------------|-----------|--------
| `replica_id` | [`text`] | The ID of a cluster replica.
| `process_id` | [`uint8`] | The ID of a process within the replica.
| `status` | [`text`] | The status of the cluster replica: `online` or `offline`.
| `reason` | [`text`] | If the cluster replica is in an `offline` state, the reason (if available). For example, `oom-killed`.
| `occurred_at` | [`timestamp with time zone`] | Wall-clock timestamp at which the event occurred.

## `mz_cluster_replica_utilization`

The `mz_cluster_replica_utilization` view gives the last known CPU and RAM utilization statistics
Expand All @@ -224,7 +239,7 @@ At this time, we do not make any guarantees about the exactness or freshness of
| Field | Type | Meaning |
|------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `replica_id` | [`text`] | The ID of a cluster replica. |
| `process_id` | [`uint8`] | An identifier of a compute process within a replica. |
| `process_id` | [`uint8`] | The ID of a process within the replica. |
| `cpu_percent` | [`double precision`] | Approximate CPU usage in percent of the total allocation. |
| `memory_percent` | [`double precision`] | Approximate RAM usage in percent of the total allocation. |
| `disk_percent` | [`double precision`] | Approximate disk usage in percent of the total allocation. |
Expand All @@ -241,7 +256,7 @@ At this time, we do not make any guarantees about the exactness or freshness of
| Field | Type | Meaning
|------------------|----------------------|--------
| `replica_id` | [`text`] | The ID of a cluster replica.
| `process_id` | [`uint8`] | An identifier of a compute process within the replica.
| `process_id` | [`uint8`] | The ID of a process within the replica.
| `cpu_percent` | [`double precision`] | Approximate CPU usage in percent of the total allocation.
| `memory_percent` | [`double precision`] | Approximate RAM usage in percent of the total allocation.
| `disk_percent` | [`double precision`] | Approximate disk usage in percent of the total allocation.
Expand Down
21 changes: 21 additions & 0 deletions src/adapter/src/coord/message_handler.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ use mz_ore::option::OptionExt;
use mz_ore::tracing::OpenTelemetryContext;
use mz_ore::{soft_assert_or_log, task};
use mz_persist_client::usage::ShardsUsageReferenced;
use mz_repr::{Datum, Row};
use mz_sql::ast::Statement;
use mz_sql::names::ResolvedIds;
use mz_sql::pure::PurifiedStatement;
use mz_storage_client::controller::IntrospectionType;
use mz_storage_types::controller::CollectionMetadata;
use opentelemetry::trace::TraceContextExt;
use rand::{rngs, Rng, SeedableRng};
Expand Down Expand Up @@ -742,6 +744,25 @@ impl Coordinator {
};

if event.status != replica_statues[&event.process_id].status {
if !self.controller.read_only() {
let offline_reason = match event.status {
ClusterStatus::Online => None,
ClusterStatus::Offline(None) => None,
ClusterStatus::Offline(Some(reason)) => Some(reason.to_string()),
};
let row = Row::pack_slice(&[
Datum::String(&event.replica_id.to_string()),
Datum::UInt64(event.process_id),
Datum::String(event.status.as_kebab_case_str()),
Datum::from(offline_reason.as_deref()),
Datum::TimestampTz(event.time.try_into().expect("must fit")),
]);
self.controller.storage.append_introspection_updates(
IntrospectionType::ReplicaStatusHistory,
vec![(row, 1)],
);
}

let old_replica_status =
ClusterReplicaStatuses::cluster_replica_status(replica_statues);
let old_process_status = replica_statues
Expand Down
1 change: 1 addition & 0 deletions src/adapter/src/flags.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ pub fn storage_config(config: &SystemVars) -> StorageParameters {
keep_n_sink_status_history_entries: config.keep_n_sink_status_history_entries(),
keep_n_privatelink_status_history_entries: config
.keep_n_privatelink_status_history_entries(),
keep_n_replica_status_history_entries: config.keep_n_replica_status_history_entries(),
upsert_rocksdb_tuning_config: {
match mz_rocksdb_types::RocksDBTuningParameters::from_parameters(
config.upsert_rocksdb_compaction_style(),
Expand Down
28 changes: 26 additions & 2 deletions src/catalog/src/builtin.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ use mz_sql::session::user::{
MZ_SUPPORT_ROLE_ID, MZ_SYSTEM_ROLE_ID, SUPPORT_USER_NAME, SYSTEM_USER_NAME,
};
use mz_storage_client::controller::IntrospectionType;
use mz_storage_client::healthcheck::REPLICA_METRICS_HISTORY_DESC;
use mz_storage_client::healthcheck::{
MZ_AWS_PRIVATELINK_CONNECTION_STATUS_HISTORY_DESC, MZ_PREPARED_STATEMENT_HISTORY_DESC,
MZ_SESSION_HISTORY_DESC, MZ_SINK_STATUS_HISTORY_DESC, MZ_SOURCE_STATUS_HISTORY_DESC,
MZ_SQL_TEXT_DESC, MZ_STATEMENT_EXECUTION_HISTORY_DESC,
MZ_SQL_TEXT_DESC, MZ_STATEMENT_EXECUTION_HISTORY_DESC, REPLICA_METRICS_HISTORY_DESC,
REPLICA_STATUS_HISTORY_DESC,
};
use mz_storage_client::statistics::{MZ_SINK_STATISTICS_RAW_DESC, MZ_SOURCE_STATISTICS_RAW_DESC};
use rand::Rng;
Expand Down Expand Up @@ -2727,6 +2727,8 @@ pub static MZ_PENDING_CLUSTER_REPLICAS: LazyLock<BuiltinTable> = LazyLock::new(|
access: vec![PUBLIC_SELECT],
});

// TODO(teskje) Remove this table in favor of `MZ_CLUSTER_REPLICA_STATUS_HISTORY`, once internal
// clients have been migrated.
pub static MZ_CLUSTER_REPLICA_STATUSES: LazyLock<BuiltinTable> = LazyLock::new(|| BuiltinTable {
name: "mz_cluster_replica_statuses",
schema: MZ_INTERNAL_SCHEMA,
Expand All @@ -2745,6 +2747,17 @@ pub static MZ_CLUSTER_REPLICA_STATUSES: LazyLock<BuiltinTable> = LazyLock::new(|
access: vec![PUBLIC_SELECT],
});

pub static MZ_CLUSTER_REPLICA_STATUS_HISTORY: LazyLock<BuiltinSource> =
LazyLock::new(|| BuiltinSource {
name: "mz_cluster_replica_status_history",
schema: MZ_INTERNAL_SCHEMA,
oid: oid::SOURCE_MZ_CLUSTER_REPLICA_STATUS_HISTORY_OID,
data_source: IntrospectionType::ReplicaStatusHistory,
desc: REPLICA_STATUS_HISTORY_DESC.clone(),
is_retained_metrics_object: false,
access: vec![PUBLIC_SELECT],
});

pub static MZ_CLUSTER_REPLICA_SIZES: LazyLock<BuiltinTable> = LazyLock::new(|| BuiltinTable {
name: "mz_cluster_replica_sizes",
schema: MZ_CATALOG_SCHEMA,
Expand Down Expand Up @@ -7655,6 +7668,15 @@ ON mz_internal.mz_cluster_replica_statuses (replica_id)",
is_retained_metrics_object: true,
};

pub const MZ_CLUSTER_REPLICA_STATUS_HISTORY_IND: BuiltinIndex = BuiltinIndex {
name: "mz_cluster_replica_status_history_ind",
schema: MZ_INTERNAL_SCHEMA,
oid: oid::INDEX_MZ_CLUSTER_REPLICA_STATUS_HISTORY_IND_OID,
sql: "IN CLUSTER mz_catalog_server
ON mz_internal.mz_cluster_replica_status_history (replica_id)",
is_retained_metrics_object: false,
};

pub const MZ_CLUSTER_REPLICA_METRICS_IND: BuiltinIndex = BuiltinIndex {
name: "mz_cluster_replica_metrics_ind",
schema: MZ_INTERNAL_SCHEMA,
Expand Down Expand Up @@ -8097,6 +8119,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
Builtin::Source(&MZ_CLUSTER_REPLICA_METRICS_HISTORY),
Builtin::Table(&MZ_CLUSTER_REPLICA_SIZES),
Builtin::Table(&MZ_CLUSTER_REPLICA_STATUSES),
Builtin::Source(&MZ_CLUSTER_REPLICA_STATUS_HISTORY),
Builtin::Table(&MZ_INTERNAL_CLUSTER_REPLICAS),
Builtin::Table(&MZ_PENDING_CLUSTER_REPLICAS),
Builtin::Table(&MZ_AUDIT_EVENTS),
Expand Down Expand Up @@ -8332,6 +8355,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
Builtin::Index(&MZ_CLUSTER_REPLICAS_IND),
Builtin::Index(&MZ_CLUSTER_REPLICA_SIZES_IND),
Builtin::Index(&MZ_CLUSTER_REPLICA_STATUSES_IND),
Builtin::Index(&MZ_CLUSTER_REPLICA_STATUS_HISTORY_IND),
Builtin::Index(&MZ_CLUSTER_REPLICA_METRICS_IND),
Builtin::Index(&MZ_CLUSTER_REPLICA_METRICS_HISTORY_IND),
Builtin::Index(&MZ_CLUSTER_REPLICA_HISTORY_IND),
Expand Down
Loading

0 comments on commit 95aa493

Please sign in to comment.