Skip to content

Commit

Permalink
cluster-ui: preliminary work moving database details endpoint
Browse files Browse the repository at this point in the history 
This change establishes preliminary work to move the existing database
details endpoint to use sql-over-http, making it tenant-scoped.
Currently, this change is missing a couple existing fields:
- `approximate_disk_bytes`, used to display the size of the database
- `node_ids`, used to display the nodes that the database uses

The data for database details is populated using 5 queries, which fetch
the database's:
- ID
- grants
- tables
- range statistics
- index usage statistics

This PR also changes the structure of the database details response.
Each field within the overlying database details response encapsulates
the response of one of the queries. Query failures are captured at the
top-level of the response and within the corresponding response field
for the query (scoping query failures to each transaction).

Release note: None
  • Loading branch information
Thomas Hardy committed Feb 13, 2023
1 parent f503e3a commit 57a5b4f
Show file tree
Hide file tree
Showing 14 changed files with 549 additions and 563 deletions.
234 changes: 126 additions & 108 deletions pkg/ui/workspaces/cluster-ui/src/api/databaseDetailsApi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,27 @@ import {
txnResultIsEmpty,
} from "./sqlApi";
import { recommendDropUnusedIndex } from "../insights";
import { Format, Identifier } from "./safesql";
import moment from "moment";
import { withTimeout } from "./util";

type DatabaseDetailsResponse = {
export type DatabaseDetailsResponse = {
grants_resp: DatabaseGrantsResponse;
tables_resp: DatabaseTablesResponse;
id_resp: DatabaseIdResponse;
stats?: DatabaseDetailsStats;
error?: SqlExecutionErrorMessage;
};

function newDatabaseDetailsResponse(): DatabaseDetailsResponse {
export function newDatabaseDetailsResponse(): DatabaseDetailsResponse {
return {
grants_resp: { grants: [] },
tables_resp: { tables: [] },
id_resp: { id: { database_id: "" } },
stats: {
ranges_data: {
count: 0,
node_ids: [],
regions: [],
},
index_stats: { num_index_recommendations: 0 },
Expand All @@ -43,15 +47,20 @@ function newDatabaseDetailsResponse(): DatabaseDetailsResponse {
}

type DatabaseDetailsStats = {
pebble_data?: DatabasePebbleData;
ranges_data: DatabaseRangesData;
index_stats: DatabaseIndexUsageStatsResponse;
};

type DatabasePebbleData = {
approximate_disk_bytes: number; // (can't get this currently via SQL)
};

type DatabaseRangesData = {
// missing_tables (not sure if this is necessary)
count?: number;
// approximate_disk_bytes: number (can't get this currently - range_size_mb in SHOW RANGES gives total uncompressed disk space - not the same... this is used for 'Live Data')
// node_ids?: number (don't think we can get this currently, SHOW RANGES only shows replica ids)
count: number;
// TODO(thomas): currently we are using replicas to populate node ids
// which does not map 1 to 1.
node_ids: number[];
regions: string[];
error?: Error;
};
Expand All @@ -65,12 +74,14 @@ type DatabaseIdRow = {
database_id: string;
};

const getDatabaseId = (dbName: string): DatabaseDetailsQuery<DatabaseIdRow> => {
const stmt: SqlStatement = {
sql: `SELECT crdb_internal.get_database_id($1) as database_id`,
arguments: [dbName],
};
const addToDatabaseDetail = (
const getDatabaseId: DatabaseDetailsQuery<DatabaseIdRow> = {
createStmt: dbName => {
return {
sql: `SELECT crdb_internal.get_database_id($1) as database_id`,
arguments: [dbName],
};
},
addToDatabaseDetail: (
txn_result: SqlTxnResult<DatabaseIdRow>,
resp: DatabaseDetailsResponse,
) => {
Expand All @@ -80,11 +91,7 @@ const getDatabaseId = (dbName: string): DatabaseDetailsQuery<DatabaseIdRow> => {
if (txn_result.error) {
resp.id_resp.error = txn_result.error;
}
};
return {
stmt,
addToDatabaseDetail,
};
},
};

type DatabaseGrantsResponse = {
Expand All @@ -99,13 +106,15 @@ type DatabaseGrantsRow = {
is_grantable: boolean;
};

const getDatabaseGrantsQuery = (
dbName: string,
): DatabaseDetailsQuery<DatabaseGrantsRow> => {
const stmt: SqlStatement = {
sql: `SELECT * FROM [SHOW GRANTS ON DATABASE ${dbName}]`,
};
const addToDatabaseDetail = (
const getDatabaseGrantsQuery: DatabaseDetailsQuery<DatabaseGrantsRow> = {
createStmt: dbName => {
return {
sql: Format(`SELECT * FROM [SHOW GRANTS ON DATABASE %1]`, [
new Identifier(dbName),
]),
};
},
addToDatabaseDetail: (
txn_result: SqlTxnResult<DatabaseGrantsRow>,
resp: DatabaseDetailsResponse,
) => {
Expand All @@ -115,11 +124,7 @@ const getDatabaseGrantsQuery = (
resp.grants_resp.error = txn_result.error;
}
}
};
return {
stmt,
addToDatabaseDetail,
};
},
};

type DatabaseTablesResponse = {
Expand All @@ -132,14 +137,17 @@ type DatabaseTablesRow = {
table_name: string;
};

const getDatabaseTablesQuery = (
dbName: string,
): DatabaseDetailsQuery<DatabaseTablesRow> => {
const stmt: SqlStatement = {
sql: `SELECT table_schema, table_name FROM ${dbName}.information_schema.tables WHERE table_catalog = $1 AND table_type != 'SYSTEM VIEW' ORDER BY table_name`,
arguments: [dbName],
};
const addToDatabaseDetail = (
const getDatabaseTablesQuery: DatabaseDetailsQuery<DatabaseTablesRow> = {
createStmt: dbName => {
return {
sql: Format(
`SELECT table_schema, table_name FROM %1.information_schema.tables WHERE table_catalog = $1 AND table_type != 'SYSTEM VIEW' ORDER BY table_name`,
[new Identifier(dbName)],
),
arguments: [dbName],
};
},
addToDatabaseDetail: (
txn_result: SqlTxnResult<DatabaseTablesRow>,
resp: DatabaseDetailsResponse,
) => {
Expand All @@ -149,11 +157,7 @@ const getDatabaseTablesQuery = (
if (txn_result.error) {
resp.tables_resp.error = txn_result.error;
}
};
return {
stmt,
addToDatabaseDetail,
};
},
};

type DatabaseRangesRow = {
Expand All @@ -167,11 +171,11 @@ type DatabaseRangesRow = {
range_size: number;
};

const getDatabaseRanges = (
dbName: string,
): DatabaseDetailsQuery<DatabaseRangesRow> => {
const stmt: SqlStatement = {
sql: `SELECT
const getDatabaseRanges: DatabaseDetailsQuery<DatabaseRangesRow> = {
createStmt: dbName => {
return {
sql: Format(
`SELECT
r.range_id,
t.table_id,
t.database_name,
Expand All @@ -184,29 +188,35 @@ const getDatabaseRanges = (
coalesce((crdb_internal.range_stats(s.start_key)->>'range_key_bytes')::INT, 0) +
coalesce((crdb_internal.range_stats(s.start_key)->>'range_val_bytes')::INT, 0) AS range_size
FROM crdb_internal.tables as t
JOIN ${dbName}.crdb_internal.table_spans as s ON s.descriptor_id = t.table_id
JOIN %1.crdb_internal.table_spans as s ON s.descriptor_id = t.table_id
JOIN crdb_internal.ranges_no_leases as r ON s.start_key < r.end_key AND s.end_key > r.start_key
WHERE t.database_name = $1`,
arguments: [dbName],
};
const addToDatabaseDetail = (
[new Identifier(dbName)],
),
arguments: [dbName],
};
},
addToDatabaseDetail: (
txn_result: SqlTxnResult<DatabaseRangesRow>,
resp: DatabaseDetailsResponse,
) => {
// Build set of unique regions for this database.
const regions = new Set<string>();
// Build set of unique replicas for this database.
const replicas = new Set<number>();
if (!txnResultIsEmpty(txn_result)) {
txn_result.rows.forEach(row => {
resp.stats.ranges_data.regions = row.regions;
row.regions.forEach(regions.add, regions);
row.replicas.forEach(replicas.add, replicas);
});
resp.stats.ranges_data.regions = Array.from(regions.values());
resp.stats.ranges_data.node_ids = Array.from(replicas.values());
resp.stats.ranges_data.count = txn_result.rows.length;
}
if (txn_result.error) {
resp.stats.ranges_data.error = txn_result.error;
}
};
return {
stmt,
addToDatabaseDetail,
};
},
};

type DatabaseIndexUsageStatsResponse = {
Expand All @@ -227,11 +237,12 @@ export type DatabaseIndexUsageStatsRow = {
unused_threshold: string;
};

const getDatabaseIndexUsageStats = (
dbName: string,
): DatabaseDetailsQuery<DatabaseIndexUsageStatsRow> => {
const stmt: SqlStatement = {
sql: `SELECT
const getDatabaseIndexUsageStats: DatabaseDetailsQuery<DatabaseIndexUsageStatsRow> =
{
createStmt: dbName => {
return {
sql: Format(
`SELECT
t.database_name,
ti.descriptor_name as table_name,
ti.descriptor_id as table_id,
Expand All @@ -242,75 +253,82 @@ const getDatabaseIndexUsageStats = (
last_read,
ti.created_at,
(SELECT value FROM crdb_internal.cluster_settings WHERE variable = 'sql.index_recommendation.drop_unused_duration') AS unused_threshold
FROM ${dbName}.crdb_internal.index_usage_statistics AS us
JOIN ${dbName}.crdb_internal.table_indexes AS ti ON (us.index_id = ti.index_id AND us.table_id = ti.descriptor_id AND index_type = 'secondary')
JOIN ${dbName}.crdb_internal.tables AS t ON (ti.descriptor_id = t.table_id AND t.database_name != 'system')`,
};
const addToDatabaseDetail = (
txn_result: SqlTxnResult<DatabaseIndexUsageStatsRow>,
resp: DatabaseDetailsResponse,
) => {
txn_result.rows?.forEach(row => {
const rec = recommendDropUnusedIndex(row);
if (rec.recommend) {
resp.stats.index_stats.num_index_recommendations += 1;
FROM %1.crdb_internal.index_usage_statistics AS us
JOIN %1.crdb_internal.table_indexes AS ti ON (us.index_id = ti.index_id AND us.table_id = ti.descriptor_id AND index_type = 'secondary')
JOIN %1.crdb_internal.tables AS t ON (ti.descriptor_id = t.table_id AND t.database_name != 'system')`,
[new Identifier(dbName)],
),
};
},
addToDatabaseDetail: (
txn_result: SqlTxnResult<DatabaseIndexUsageStatsRow>,
resp: DatabaseDetailsResponse,
) => {
txn_result.rows?.forEach(row => {
const rec = recommendDropUnusedIndex(row);
if (rec.recommend) {
resp.stats.index_stats.num_index_recommendations += 1;
}
});
if (txn_result.error) {
resp.stats.index_stats.error = txn_result.error;
}
});
if (txn_result.error) {
resp.stats.index_stats.error = txn_result.error;
}
};
return {
stmt,
addToDatabaseDetail,
},
};
};

type DatabaseDetailsRow =
export type DatabaseDetailsRow =
| DatabaseIdRow
| DatabaseGrantsRow
| DatabaseTablesRow
| DatabaseRangesRow
| DatabaseIndexUsageStatsRow;

type DatabaseDetailsQuery<RowType> = {
stmt: SqlStatement;
createStmt: (dbName: string) => SqlStatement;
addToDatabaseDetail: (
response: SqlTxnResult<RowType>,
dbDetail: DatabaseDetailsResponse,
) => void;
};

export async function getDatabaseDetails(databaseName: string) {
const databaseDetailQueries: DatabaseDetailsQuery<DatabaseDetailsRow>[] = [
getDatabaseId(databaseName),
getDatabaseGrantsQuery(databaseName),
getDatabaseTablesQuery(databaseName),
getDatabaseRanges(databaseName),
getDatabaseIndexUsageStats(databaseName),
];
const databaseDetailQueries: DatabaseDetailsQuery<DatabaseDetailsRow>[] = [
getDatabaseId,
getDatabaseGrantsQuery,
getDatabaseTablesQuery,
getDatabaseRanges,
getDatabaseIndexUsageStats,
];

const req: SqlExecutionRequest = {
export function createDatabaseDetailsReq(dbName: string): SqlExecutionRequest {
return {
execute: true,
statements: databaseDetailQueries.map(query => query.stmt),
statements: databaseDetailQueries.map(query => query.createStmt(dbName)),
max_result_size: LARGE_RESULT_SIZE,
};
}

export async function getDatabaseDetails(
databaseName: string,
timeout?: moment.Duration,
): Promise<DatabaseDetailsResponse> {
const req: SqlExecutionRequest = createDatabaseDetailsReq(databaseName);
const resp: DatabaseDetailsResponse = newDatabaseDetailsResponse();

const res = await executeInternalSql<DatabaseDetailsRow>(req);

res.execution.txn_results.forEach(txn_result => {
if (txn_result.rows) {
const query: DatabaseDetailsQuery<DatabaseDetailsRow> =
databaseDetailQueries[txn_result.statement - 1];
query.addToDatabaseDetail(txn_result, resp);
}
});
return withTimeout(executeInternalSql<DatabaseDetailsRow>(req), timeout).then(
res => {
res.execution.txn_results.forEach(txn_result => {
if (txn_result.rows) {
const query: DatabaseDetailsQuery<DatabaseDetailsRow> =
databaseDetailQueries[txn_result.statement - 1];
query.addToDatabaseDetail(txn_result, resp);
}
});

if (res.error) {
resp.error = res.error;
}
if (res.error) {
resp.error = res.error;
}

return resp;
return resp;
},
);
}
1 change: 1 addition & 0 deletions pkg/ui/workspaces/cluster-ui/src/api/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ export * from "./sqlApi";
export * from "./tracezApi";
export * from "./databasesApi";
export * from "./eventsApi";
export * from "./databaseDetailsApi";
Loading

0 comments on commit 57a5b4f

Please sign in to comment.