From a01fa4db89286382f6ad2441fb21bee24ee34f47 Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Fri, 1 Jul 2022 17:02:33 +0100 Subject: [PATCH 1/4] roachtest: run workload from the tenant node The secure URL refers to paths on disk on the clusters in the node. Since we only create the tenant-scoped certs on the tenant node, we need to run workload from that node. Fixes #82266 Depends on #83703 Release note: None --- pkg/cmd/roachtest/tests/smoketest_secure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cmd/roachtest/tests/smoketest_secure.go b/pkg/cmd/roachtest/tests/smoketest_secure.go index d0ca5065c20f..a0d7465947fc 100644 --- a/pkg/cmd/roachtest/tests/smoketest_secure.go +++ b/pkg/cmd/roachtest/tests/smoketest_secure.go @@ -80,7 +80,7 @@ func multitenantSmokeTest(ctx context.Context, t test.Test, c cluster.Cluster) { // init kv and check new database was done right cmd := fmt.Sprintf("./cockroach workload init kv '%s'", ten.secureURL()) - err = c.RunE(ctx, c.Node(1), cmd) + err = c.RunE(ctx, c.Node(2), cmd) require.NoError(t, err) sqlutils.MakeSQLRunner(db).CheckQueryResultsRetry(t, fmt.Sprintf(` From 15ec7803926eac8bb81806977b3e769347abf4a7 Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Thu, 7 Jul 2022 10:48:33 -0400 Subject: [PATCH 2/4] storage: close pebble iter gracefully when NewPebbleSSTIterator fails Currently, if `pebble.NewExternalIter` sets pebbleIterator.inuse to True, but then fails, the subsequent `pebbleIterator.destroy()` will panic unecessarily, since the caller of `pebble.NewExternalIter` is not actually using the iter. This bug causes TestBackupRestoreChecksum to flake in #83984. To fix, this patch uses pebble.Close() to gracefully close the pebbleIterator if `pebble.NewExternalIter` fails. Release Note: None --- pkg/storage/pebble_iterator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storage/pebble_iterator.go b/pkg/storage/pebble_iterator.go index ae562f498afb..8ea1a99477bb 100644 --- a/pkg/storage/pebble_iterator.go +++ b/pkg/storage/pebble_iterator.go @@ -106,7 +106,7 @@ func newPebbleSSTIterator(files []sstable.ReadableFile, opts IterOptions) (*pebb var err error if p.iter, err = pebble.NewExternalIter(DefaultPebbleOptions(), &p.options, files); err != nil { - p.destroy() + p.Close() return nil, err } return p, nil From c9f84743da635472642c8f5393740216fcf8ac91 Mon Sep 17 00:00:00 2001 From: Eric Harmeling Date: Thu, 16 Jun 2022 16:10:06 -0400 Subject: [PATCH 3/4] ui: add internal app filter to active statements and transactions pages This commit adds a single internal app filter option on to the Active Statements and Active Transactions pages. Active statements and transactions run by internal apps are no longer displayed by default. Release note (ui change): The Active Statements and Active Transactions pages now have a single filter option for internal apps. These pages no longer display internal statements and transactions by default. --- .../activeStatementUtils.spec.ts | 68 ++++++++++++---- .../activeExecutions/activeStatementUtils.ts | 81 ++++++++++++++++--- .../activeStatementsPage.selectors.ts | 9 +++ .../statementsPage/activeStatementsView.tsx | 5 +- .../activeTransactionsPage.selectors.tsx | 2 + .../activeTransactionsView.tsx | 8 +- .../statements/activeStatementsSelectors.tsx | 11 +++ .../activeTransactionsSelectors.tsx | 2 + 8 files changed, 155 insertions(+), 31 deletions(-) diff --git a/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.spec.ts b/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.spec.ts index 584a5e836ed4..867803f9980f 100644 --- a/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.spec.ts +++ b/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.spec.ts @@ -14,6 +14,8 @@ import { ActiveTransaction, ActiveStatement, SessionStatusType, + ActiveStatementFilters, + ActiveTransactionFilters, } from "./types"; import * as protos from "@cockroachlabs/crdb-protobuf-client"; import moment from "moment"; @@ -26,6 +28,7 @@ import { getActiveStatementsFromSessions, filterActiveStatements, filterActiveTransactions, + INTERNAL_APP_NAME_PREFIX, } from "./activeStatementUtils"; type ActiveQuery = protos.cockroach.server.serverpb.ActiveQuery; @@ -108,9 +111,12 @@ describe("test activeStatementUtils", () => { makeActiveStatement({ executionID: "4", application: "app1" }), ]; - const filters = { app: "app1" }; - const filtered = filterActiveStatements(statements, filters); - + const filters: ActiveStatementFilters = { app: "app1" }; + const filtered = filterActiveStatements( + statements, + filters, + INTERNAL_APP_NAME_PREFIX, + ); expect(filtered.length).toBe(2); expect(filtered[0].executionID).toBe("1"); expect(filtered[1].executionID).toBe("4"); @@ -140,9 +146,14 @@ describe("test activeStatementUtils", () => { }), ]; - const filters = { app: "app1" }; + const filters: ActiveStatementFilters = { app: "app1" }; const search = "SELECT 1"; - const filtered = filterActiveStatements(statements, filters, search); + const filtered = filterActiveStatements( + statements, + filters, + INTERNAL_APP_NAME_PREFIX, + search, + ); expect(filtered.length).toBe(2); expect(filtered[0].executionID).toBe("1"); @@ -158,8 +169,12 @@ describe("test activeStatementUtils", () => { makeActiveStatement(), ]; - const filters = { app: "" }; - const filtered = filterActiveStatements(statements, filters, ""); + const filters: ActiveStatementFilters = {}; + const filtered = filterActiveStatements( + statements, + filters, + INTERNAL_APP_NAME_PREFIX, + ); expect(filtered.length).toBe(statements.length); }); @@ -196,7 +211,7 @@ describe("test activeStatementUtils", () => { }, ], errors: [], - internal_app_name_prefix: "", + internal_app_name_prefix: INTERNAL_APP_NAME_PREFIX, toJSON: () => ({}), }; @@ -237,7 +252,10 @@ describe("test activeStatementUtils", () => { makeActiveStatement({ application: "app3" }), makeActiveStatement({ application: "app4" }), ]; - const apps = getAppsFromActiveStatements(activeStatements); + const apps = getAppsFromActiveStatements( + activeStatements, + INTERNAL_APP_NAME_PREFIX, + ); expect(apps).toEqual(["app1", "app2", "app3", "app4"]); }); @@ -290,7 +308,7 @@ describe("test activeStatementUtils", () => { }, ], errors: [], - internal_app_name_prefix: "", + internal_app_name_prefix: INTERNAL_APP_NAME_PREFIX, toJSON: () => ({}), }; @@ -328,8 +346,12 @@ describe("test activeStatementUtils", () => { makeActiveTxn({ executionID: "4", application: "app1" }), ]; - const filters = { app: "app1" }; - const filtered = filterActiveTransactions(txns, filters); + const filters: ActiveTransactionFilters = { app: "app1" }; + const filtered = filterActiveTransactions( + txns, + filters, + INTERNAL_APP_NAME_PREFIX, + ); expect(filtered.length).toBe(2); expect(filtered[0].executionID).toBe("1"); @@ -359,9 +381,14 @@ describe("test activeStatementUtils", () => { }), ]; - const filters = { app: "app1" }; + const filters: ActiveTransactionFilters = { app: "app1" }; const search = "SELECT 1"; - const filtered = filterActiveTransactions(txns, filters, search); + const filtered = filterActiveTransactions( + txns, + filters, + INTERNAL_APP_NAME_PREFIX, + search, + ); expect(filtered.length).toBe(2); expect(filtered[0].executionID).toBe("1"); @@ -377,8 +404,12 @@ describe("test activeStatementUtils", () => { makeActiveTxn(), ]; - const filters = { app: "" }; - const filtered = filterActiveTransactions(txns, filters, ""); + const filters: ActiveTransactionFilters = {}; + const filtered = filterActiveTransactions( + txns, + filters, + INTERNAL_APP_NAME_PREFIX, + ); expect(filtered.length).toBe(txns.length); }); @@ -392,7 +423,10 @@ describe("test activeStatementUtils", () => { makeActiveTxn({ application: "app4" }), ]; - const apps = getAppsFromActiveTransactions(activeTxns); + const apps = getAppsFromActiveTransactions( + activeTxns, + INTERNAL_APP_NAME_PREFIX, + ); expect(apps).toEqual(["app1", "app2", "app3", "app4"]); }); }); diff --git a/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.ts b/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.ts index dd016bcd8b02..931f03f3239c 100644 --- a/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.ts +++ b/pkg/ui/workspaces/cluster-ui/src/activeExecutions/activeStatementUtils.ts @@ -10,7 +10,7 @@ import moment, { Moment } from "moment"; import { byteArrayToUuid } from "src/sessions"; -import { TimestampToMoment } from "src/util"; +import { TimestampToMoment, unset } from "src/util"; import { ActiveTransaction } from "."; import { SessionsResponse, @@ -22,17 +22,42 @@ import { import { ActiveStatement, ActiveStatementFilters } from "./types"; export const ACTIVE_STATEMENT_SEARCH_PARAM = "q"; +export const INTERNAL_APP_NAME_PREFIX = "$ internal"; export function filterActiveStatements( statements: ActiveStatement[], filters: ActiveStatementFilters, + internalAppNamePrefix: string, search?: string, ): ActiveStatement[] { if (statements == null) return []; - let filteredStatements = statements.filter( - stmt => filters.app === "" || stmt.application === filters.app, - ); + let filteredStatements = statements; + + const isInternal = (statement: ActiveStatement) => + statement.application.startsWith(internalAppNamePrefix); + if (filters.app) { + filteredStatements = filteredStatements.filter( + (statement: ActiveStatement) => { + const apps = filters.app.toString().split(","); + let showInternal = false; + if (apps.includes(internalAppNamePrefix)) { + showInternal = true; + } + if (apps.includes(unset)) { + apps.push(""); + } + return ( + (showInternal && isInternal(statement)) || + apps.includes(statement.application) + ); + }, + ); + } else { + filteredStatements = filteredStatements.filter( + statement => !isInternal(statement), + ); + } if (search) { filteredStatements = filteredStatements.filter(stmt => @@ -84,27 +109,51 @@ export function getActiveStatementsFromSessions( export function getAppsFromActiveStatements( statements: ActiveStatement[] | null, + internalAppNamePrefix: string, ): string[] { if (statements == null) return []; - return Array.from( - statements.reduce( - (apps, stmt) => apps.add(stmt.application), - new Set(), - ), + + const uniqueAppNames = new Set( + statements.map(s => { + if (s.application.startsWith(internalAppNamePrefix)) { + return internalAppNamePrefix; + } + return s.application ? s.application : unset; + }), ); + + return Array.from(uniqueAppNames).sort(); } export function filterActiveTransactions( txns: ActiveTransaction[] | null, filters: ActiveTransactionFilters, + internalAppNamePrefix: string, search?: string, ): ActiveTransaction[] { if (txns == null) return []; let filteredTxns = txns; + const isInternal = (txn: ActiveTransaction) => + txn.application.startsWith(internalAppNamePrefix); if (filters.app) { - filteredTxns = filteredTxns.filter(txn => txn.application === filters.app); + filteredTxns = filteredTxns.filter((txn: ActiveTransaction) => { + const apps = filters.app.toString().split(","); + let showInternal = false; + if (apps.includes(internalAppNamePrefix)) { + showInternal = true; + } + if (apps.includes(unset)) { + apps.push(""); + } + + return ( + (showInternal && isInternal(txn)) || apps.includes(txn.application) + ); + }); + } else { + filteredTxns = filteredTxns.filter(txn => !isInternal(txn)); } if (search) { @@ -118,12 +167,20 @@ export function filterActiveTransactions( export function getAppsFromActiveTransactions( txns: ActiveTransaction[], + internalAppNamePrefix: string, ): string[] { if (txns == null) return []; - return Array.from( - txns.reduce((apps, txn) => apps.add(txn.application), new Set()), + const uniqueAppNames = new Set( + txns.map(t => { + if (t.application.startsWith(internalAppNamePrefix)) { + return internalAppNamePrefix; + } + return t.application ? t.application : unset; + }), ); + + return Array.from(uniqueAppNames).sort(); } export function getActiveTransactionsFromSessions( diff --git a/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsPage.selectors.ts b/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsPage.selectors.ts index 0ad4e8c13e49..e9e5664e2e83 100644 --- a/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsPage.selectors.ts +++ b/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsPage.selectors.ts @@ -31,6 +31,14 @@ export const selectActiveStatements = createSelector( }, ); +export const selectAppName = createSelector( + (state: AppState) => state.adminUI.sessions, + response => { + if (!response.data) return null; + return response.data.internal_app_name_prefix; + }, +); + export const selectSortSetting = (state: AppState): SortSetting => localStorageSelector(state)["sortSetting/ActiveStatementsPage"]; @@ -59,6 +67,7 @@ export const mapStateToActiveStatementsPageProps = ( selectedColumns: selectColumns(state), sortSetting: selectSortSetting(state), filters: selectFilters(state), + internalAppNamePrefix: selectAppName(state), }); export const mapDispatchToActiveStatementsPageProps = ( diff --git a/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsView.tsx b/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsView.tsx index f93508f3697e..ffea18d1f37f 100644 --- a/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsView.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsView.tsx @@ -53,6 +53,7 @@ export type ActiveStatementsViewStateProps = { sortSetting: SortSetting; sessionsError: Error | null; filters: ActiveStatementFilters; + internalAppNamePrefix: string; }; export type ActiveStatementsViewProps = ActiveStatementsViewStateProps & @@ -68,6 +69,7 @@ export const ActiveStatementsView: React.FC = ({ statements, sessionsError, filters, + internalAppNamePrefix, }: ActiveStatementsViewProps) => { const [pagination, setPagination] = useState({ current: 1, @@ -152,12 +154,13 @@ export const ActiveStatementsView: React.FC = ({ const clearSearch = () => onSubmitSearch(""); const clearFilters = () => onSubmitFilters({ app: inactiveFiltersState.app }); - const apps = getAppsFromActiveStatements(statements); + const apps = getAppsFromActiveStatements(statements, internalAppNamePrefix); const countActiveFilters = calculateActiveFilters(filters); const filteredStatements = filterActiveStatements( statements, filters, + internalAppNamePrefix, search, ); diff --git a/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsPage.selectors.tsx b/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsPage.selectors.tsx index cf99ae86ce41..43be20014427 100644 --- a/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsPage.selectors.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsPage.selectors.tsx @@ -11,6 +11,7 @@ import { createSelector } from "reselect"; import { getActiveTransactionsFromSessions } from "../activeExecutions/activeStatementUtils"; import { localStorageSelector } from "src/statementsPage/statementsPage.selectors"; +import { selectAppName } from "src/statementsPage/activeStatementsPage.selectors"; import { ActiveTransactionFilters, ActiveTransactionsViewDispatchProps, @@ -62,6 +63,7 @@ export const mapStateToActiveTransactionsPageProps = ( selectedColumns: selectColumns(state), sortSetting: selectSortSetting(state), filters: selectFilters(state), + internalAppNamePrefix: selectAppName(state), }); export const mapDispatchToActiveTransactionsPageProps = ( diff --git a/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsView.tsx b/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsView.tsx index ec34279474fe..52384024ab61 100644 --- a/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsView.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/transactionsPage/activeTransactionsView.tsx @@ -53,6 +53,7 @@ export type ActiveTransactionsViewStateProps = { sessionsError: Error | null; filters: ActiveTransactionFilters; sortSetting: SortSetting; + internalAppNamePrefix: string; }; export type ActiveTransactionsViewProps = ActiveTransactionsViewStateProps & @@ -70,6 +71,7 @@ export const ActiveTransactionsView: React.FC = ({ transactions, sessionsError, filters, + internalAppNamePrefix, }: ActiveTransactionsViewProps) => { const [pagination, setPagination] = useState({ current: 1, @@ -152,12 +154,16 @@ export const ActiveTransactionsView: React.FC = ({ const clearSearch = () => onSubmitSearch(""); const clearFilters = () => onSubmitFilters({ app: inactiveFiltersState.app }); - const apps = getAppsFromActiveTransactions(transactions); + const apps = getAppsFromActiveTransactions( + transactions, + internalAppNamePrefix, + ); const countActiveFilters = calculateActiveFilters(filters); const filteredTransactions = filterActiveTransactions( transactions, filters, + internalAppNamePrefix, search, ); return ( diff --git a/pkg/ui/workspaces/db-console/src/views/statements/activeStatementsSelectors.tsx b/pkg/ui/workspaces/db-console/src/views/statements/activeStatementsSelectors.tsx index e753d715180e..f9256563c41e 100644 --- a/pkg/ui/workspaces/db-console/src/views/statements/activeStatementsSelectors.tsx +++ b/pkg/ui/workspaces/db-console/src/views/statements/activeStatementsSelectors.tsx @@ -32,6 +32,16 @@ const selectActiveStatements = createSelector( }, ); +export const selectAppName = createSelector( + (state: AdminUIState) => state.cachedData.sessions, + (state?: CachedDataReducerState) => { + if (!state.data) { + return null; + } + return state.data.internal_app_name_prefix; + }, +); + const selectedColumnsLocalSetting = new LocalSetting< AdminUIState, string | null @@ -62,6 +72,7 @@ export const mapStateToActiveStatementViewProps = (state: AdminUIState) => ({ sortSetting: sortSettingLocalSetting.selector(state), statements: selectActiveStatements(state), sessionsError: state.cachedData?.sessions.lastError, + internalAppNamePrefix: selectAppName(state), }); export const activeStatementsViewActions = { diff --git a/pkg/ui/workspaces/db-console/src/views/transactions/activeTransactionsSelectors.tsx b/pkg/ui/workspaces/db-console/src/views/transactions/activeTransactionsSelectors.tsx index 30a2f38ea409..424a44060f1a 100644 --- a/pkg/ui/workspaces/db-console/src/views/transactions/activeTransactionsSelectors.tsx +++ b/pkg/ui/workspaces/db-console/src/views/transactions/activeTransactionsSelectors.tsx @@ -21,6 +21,7 @@ import { LocalSetting } from "src/redux/localsettings"; import { AdminUIState } from "src/redux/state"; import { SessionsResponseMessage } from "src/util/api"; import { refreshSessions } from "src/redux/apiReducers"; +import { selectAppName } from "src/views/statements/activeStatementsSelectors"; const selectActiveTransactions = createSelector( (state: AdminUIState) => state.cachedData.sessions, @@ -62,6 +63,7 @@ export const mapStateToActiveTransactionsPageProps = (state: AdminUIState) => ({ sessionsError: state.cachedData?.sessions.lastError, filters: filtersLocalSetting.selector(state), sortSetting: sortSettingLocalSetting.selector(state), + internalAppNamePrefix: selectAppName(state), }); export const activeTransactionsPageActions = { From b6d9372720c9f03fa1aabfa7db55bafc399e393e Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Thu, 7 Jul 2022 17:11:18 -0400 Subject: [PATCH 4/4] prometheus: use older node_exporter v1.3.1, the most up to date released version, has a bug that inflates the bytes written by ~8x for NVMe drives (which in particular includes the default drives for our GCE roachprod machines). Fundamentally this is caused by the fact that these devices use a 4K sector size whereas the kernel will always report based on a 512B sector size. This took us a while to figure out, and to avoid repeating this exercise periodically, downgrade node_exporter to 1.2.2, which pre-dates a refactor that introduces the regression. See: https://github.com/prometheus/node_exporter/issues/2310 Release note: None --- pkg/roachprod/prometheus/prometheus.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/roachprod/prometheus/prometheus.go b/pkg/roachprod/prometheus/prometheus.go index 60456bc16c26..ade9566c6044 100644 --- a/pkg/roachprod/prometheus/prometheus.go +++ b/pkg/roachprod/prometheus/prometheus.go @@ -192,12 +192,15 @@ func Init( ctx context.Context, l *logger.Logger, c *install.SyncedCluster, cfg Config, ) (_ *Prometheus, _ error) { if len(cfg.NodeExporter) > 0 { + // NB: when upgrading here, make sure to target a version that picks up this PR: + // https://github.com/prometheus/node_exporter/pull/2311 + // At time of writing, there hasn't been a release in over half a year. if err := c.RepeatRun(ctx, l, os.Stdout, os.Stderr, cfg.NodeExporter, "download node exporter", ` (sudo systemctl stop node_exporter || true) && rm -rf node_exporter && mkdir -p node_exporter && curl -fsSL \ - https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz | + https://github.com/prometheus/node_exporter/releases/download/v1.2.2/node_exporter-1.2.2.linux-amd64.tar.gz | tar zxv --strip-components 1 -C node_exporter `); err != nil { return nil, err