From 60ae3935c9726bd8e9777e62217e26fd11f89d4c Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 6 Mar 2023 09:56:30 -0600 Subject: [PATCH 01/40] Move trends inside of custom --- extension/src/plots/model/collect.ts | 98 +++++++++++++++--- extension/src/plots/model/custom.ts | 23 +++++ extension/src/plots/model/index.ts | 87 ++++++++++++---- extension/src/plots/model/quickPick.ts | 89 ++++++++++++++--- extension/src/plots/webview/contract.ts | 46 ++++++--- extension/src/plots/webview/messages.ts | 99 ++++++++++++++++--- extension/src/vscode/title.ts | 1 + webview/src/plots/components/Plots.tsx | 7 +- .../components/customPlots/CustomPlot.tsx | 43 ++++++-- .../customPlots/customPlotsSlice.ts | 4 +- .../src/plots/components/customPlots/util.ts | 94 +++++++++++++++++- 11 files changed, 497 insertions(+), 94 deletions(-) create mode 100644 extension/src/plots/model/custom.ts diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index e2aa5dc095..53cb005736 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -2,7 +2,10 @@ import omit from 'lodash.omit' import get from 'lodash.get' import { TopLevelSpec } from 'vega-lite' import { VisualizationSpec } from 'react-vega' -import { CustomPlotsOrderValue } from '.' +import { + CustomPlotsOrderValue, + isCustomPlotOrderCheckpointValue +} from './custom' import { getRevisionFirstThreeColumns } from './util' import { ColorScale, @@ -16,7 +19,9 @@ import { TemplatePlotSection, PlotsType, Revision, - CustomPlotData + CustomPlotType, + CustomPlot, + MetricVsParamPlot } from '../webview/contract' import { EXPERIMENT_WORKSPACE_ID, @@ -242,26 +247,81 @@ export const collectCheckpointPlotsData = ( const plotsData: CheckpointPlot[] = [] for (const [key, value] of acc.plots.entries()) { - plotsData.push({ id: decodeColumn(key), values: value }) + plotsData.push({ + id: decodeColumn(key), + metric: decodeColumn(key), + type: CustomPlotType.CHECKPOINT, + values: value + }) } return plotsData } -export const getCustomPlotId = (metric: string, param: string) => - `custom-${metric}-${param}` +export const getCustomPlotId = (plot: CustomPlotsOrderValue) => + plot.type === CustomPlotType.CHECKPOINT + ? `custom-${plot.metric}` + : `custom-${plot.metric}-${plot.param}` -const collectCustomPlotData = ( +export const collectCustomCheckpointPlotData = ( + data: ExperimentsOutput +): { [metric: string]: CheckpointPlot } => { + const acc = { + iterations: {}, + plots: new Map() + } + + for (const { baseline, ...experimentsObject } of Object.values( + omit(data, EXPERIMENT_WORKSPACE_ID) + )) { + const commit = transformExperimentData(baseline) + + if (commit) { + collectFromExperimentsObject(acc, experimentsObject) + } + } + + const plotsData: { [metric: string]: CheckpointPlot } = {} + if (acc.plots.size === 0) { + return plotsData + } + + for (const [key, value] of acc.plots.entries()) { + const decodedMetric = decodeColumn(key) + plotsData[decodedMetric] = { + id: getCustomPlotId({ + metric: decodedMetric, + type: CustomPlotType.CHECKPOINT + }), + metric: decodedMetric, + type: CustomPlotType.CHECKPOINT, + values: value + } + } + + return plotsData +} + +export const isCheckpointPlot = (plot: CustomPlot): plot is CheckpointPlot => { + return plot.type === CustomPlotType.CHECKPOINT +} + +const collectMetricVsParamPlotData = ( metric: string, param: string, experiments: Experiment[] -): CustomPlotData => { +): MetricVsParamPlot => { const splitUpMetricPath = splitColumnPath(metric) const splitUpParamPath = splitColumnPath(param) - const plotData: CustomPlotData = { - id: getCustomPlotId(metric, param), + const plotData: MetricVsParamPlot = { + id: getCustomPlotId({ + metric, + param, + type: CustomPlotType.METRIC_VS_PARAM + }), metric: metric.slice(ColumnType.METRICS.length + 1), param: param.slice(ColumnType.PARAMS.length + 1), + type: CustomPlotType.METRIC_VS_PARAM, values: [] } @@ -281,13 +341,23 @@ const collectCustomPlotData = ( return plotData } +// TBD it will probably be easier and/or faster to get the data from +// experiments vs the output... export const collectCustomPlotsData = ( - metricsAndParams: CustomPlotsOrderValue[], + plotsOrderValue: CustomPlotsOrderValue[], + checkpointPlots: { [metric: string]: CheckpointPlot }, experiments: Experiment[] -): CustomPlotData[] => { - return metricsAndParams.map(({ metric, param }) => - collectCustomPlotData(metric, param, experiments) - ) +): CustomPlot[] => { + return plotsOrderValue + .map((plotOrderValue): CustomPlot => { + if (isCustomPlotOrderCheckpointValue(plotOrderValue)) { + const { metric } = plotOrderValue + return checkpointPlots[metric.slice(ColumnType.METRICS.length + 1)] + } + const { metric, param } = plotOrderValue + return collectMetricVsParamPlotData(metric, param, experiments) + }) + .filter(Boolean) } type MetricOrderAccumulator = { diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts new file mode 100644 index 0000000000..1da1f06f3b --- /dev/null +++ b/extension/src/plots/model/custom.ts @@ -0,0 +1,23 @@ +import { CustomPlotType } from '../webview/contract' + +// these names are way too lengthy +export type CustomPlotOrderCheckpointValue = { + type: CustomPlotType.CHECKPOINT + metric: string +} + +export type CustomPlotOrderMetricVsParamValue = { + type: CustomPlotType.METRIC_VS_PARAM + metric: string + param: string +} + +export type CustomPlotsOrderValue = + | CustomPlotOrderCheckpointValue + | CustomPlotOrderMetricVsParamValue + +export const isCustomPlotOrderCheckpointValue = ( + plot: CustomPlotsOrderValue +): plot is CustomPlotOrderCheckpointValue => { + return plot.type === CustomPlotType.CHECKPOINT +} diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index ebc3044cb7..aaf6f9d747 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -13,9 +13,12 @@ import { collectCommitRevisionDetails, collectOverrideRevisionDetails, collectCustomPlotsData, - getCustomPlotId + getCustomPlotId, + collectCustomCheckpointPlotData, + isCheckpointPlot } from './collect' import { getRevisionFirstThreeColumns } from './util' +import { CustomPlotsOrderValue } from './custom' import { CheckpointPlot, CheckpointPlotData, @@ -28,7 +31,9 @@ import { SectionCollapsed, CustomPlotData, PlotNumberOfItemsPerRow, - DEFAULT_HEIGHT + DEFAULT_HEIGHT, + CustomPlotsData, + CustomPlot } from '../webview/contract' import { ExperimentsOutput, @@ -50,8 +55,6 @@ import { } from '../multiSource/collect' import { isDvcError } from '../../cli/dvc/reader' -export type CustomPlotsOrderValue = { metric: string; param: string } - export class PlotsModel extends ModelWithPersistence { private readonly experiments: Experiments @@ -72,7 +75,10 @@ export class PlotsModel extends ModelWithPersistence { private multiSourceEncoding: MultiSourceEncoding = {} private checkpointPlots?: CheckpointPlot[] - private customPlots?: CustomPlotData[] + // TBD we need to move this type to a named type if + // we plan to keep this + private customCheckpointPlots?: { [metric: string]: CheckpointPlot } + private customPlots?: CustomPlot[] private selectedMetrics?: string[] private metricOrder: string[] @@ -115,7 +121,7 @@ export class PlotsModel extends ModelWithPersistence { this.setMetricOrder() - this.recreateCustomPlots() + this.recreateCustomPlots(data) return this.removeStaleData() } @@ -188,25 +194,40 @@ export class PlotsModel extends ModelWithPersistence { } } - public getCustomPlots() { + public getCustomPlots(): CustomPlotsData | undefined { if (!this.customPlots) { return } + + // TBD colors is undefined if there are no selected exps + const colors = getColorScale( + this.experiments + .getSelectedExperiments() + .map(({ displayColor, id: revision }) => ({ displayColor, revision })) + ) + return { + colors, height: this.getHeight(Section.CUSTOM_PLOTS), nbItemsPerRow: this.getNbItemsPerRow(Section.CUSTOM_PLOTS), - plots: this.customPlots + plots: this.getCustomPlotData(this.customPlots, colors?.domain) } } - public recreateCustomPlots() { + public recreateCustomPlots(data?: ExperimentsOutput) { + if (data) { + this.customCheckpointPlots = collectCustomCheckpointPlotData(data) + } const experiments = this.experiments.getExperiments() + // TBD this if check is going to nned to be rethought since checkpoint data + // is involved now if (experiments.length === 0) { this.customPlots = undefined return } - const customPlots: CustomPlotData[] = collectCustomPlotsData( + const customPlots: CustomPlot[] = collectCustomPlotsData( this.getCustomPlotsOrder(), + this.customCheckpointPlots || {}, experiments ) this.customPlots = customPlots @@ -224,16 +245,14 @@ export class PlotsModel extends ModelWithPersistence { public removeCustomPlots(plotIds: string[]) { const newCustomPlotsOrder = this.getCustomPlotsOrder().filter( - ({ metric, param }) => { - return !plotIds.includes(getCustomPlotId(metric, param)) - } + plot => !plotIds.includes(getCustomPlotId(plot)) ) this.setCustomPlotsOrder(newCustomPlotsOrder) } - public addCustomPlot(metricAndParam: CustomPlotsOrderValue) { - const newCustomPlotsOrder = [...this.getCustomPlotsOrder(), metricAndParam] + public addCustomPlot(value: CustomPlotsOrderValue) { + const newCustomPlotsOrder = [...this.getCustomPlotsOrder(), value] this.setCustomPlotsOrder(newCustomPlotsOrder) } @@ -513,22 +532,48 @@ export class PlotsModel extends ModelWithPersistence { return reorderObjectList( this.metricOrder, checkpointPlots.map(plot => { - const { id, values } = plot + const { id, values, type } = plot return { id, - title: truncateVerticalTitle( - id, - this.getNbItemsPerRow(Section.CHECKPOINT_PLOTS) - ) as string, + metric: id, + type, values: values.filter(value => selectedExperiments.includes(value.group) - ) + ), + yTitle: truncateVerticalTitle( + id, + this.getNbItemsPerRow(Section.CHECKPOINT_PLOTS) + ) as string } }), 'id' ) } + private getCustomPlotData( + plots: CustomPlot[], + selectedExperiments: string[] | undefined + ): CustomPlotData[] { + if (!selectedExperiments) { + return plots.filter(plot => !isCheckpointPlot(plot)) as CustomPlotData[] + } + return plots.map( + plot => + ({ + ...plot, + values: isCheckpointPlot(plot) + ? plot.values.filter(value => + selectedExperiments.includes(value.group) + ) + : plot.values, + yTitle: truncateVerticalTitle( + plot.metric, + this.getNbItemsPerRow(Section.CUSTOM_PLOTS) + ) as string + } as CustomPlotData) + ) + } + private getSelectedComparisonPlots( paths: string[], selectedRevisions: string[] diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index eee9723a3e..5b4f3583d8 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -1,15 +1,46 @@ -import { CustomPlotsOrderValue } from '.' import { getCustomPlotId } from './collect' +import { CustomPlotsOrderValue } from './custom' import { splitColumnPath } from '../../experiments/columns/paths' import { pickFromColumnLikes } from '../../experiments/columns/quickPick' import { Column, ColumnType } from '../../experiments/webview/contract' import { definedAndNonEmpty } from '../../util/array' import { quickPickManyValues, + quickPickValue, QuickPickOptionsWithTitle } from '../../vscode/quickPick' import { Title } from '../../vscode/title' import { Toast } from '../../vscode/toast' +import { CustomPlotType } from '../webview/contract' + +const getMetricVsParamPlotItem = (metric: string, param: string) => { + const splitMetric = splitColumnPath(metric) + const splitParam = splitColumnPath(param) + return { + description: 'Metric Vs Param Plot', + detail: `${metric} vs ${param}`, + label: `${splitMetric[splitMetric.length - 1]} vs ${ + splitParam[splitParam.length - 1] + }`, + value: getCustomPlotId({ + metric, + param, + type: CustomPlotType.METRIC_VS_PARAM + }) + } +} +const getCheckpointPlotItem = (metric: string) => { + const splitMetric = splitColumnPath(metric) + return { + description: 'Trend Plot', + detail: metric, + label: splitMetric[splitMetric.length - 1], + value: getCustomPlotId({ + metric, + type: CustomPlotType.CHECKPOINT + }) + } +} export const pickCustomPlots = ( plots: CustomPlotsOrderValue[], @@ -20,21 +51,37 @@ export const pickCustomPlots = ( return Toast.showError(noPlotsErrorMessage) } - const plotsItems = plots.map(({ metric, param }) => { - const splitMetric = splitColumnPath(metric) - const splitParam = splitColumnPath(param) - return { - description: `${metric} vs ${param}`, - label: `${splitMetric[splitMetric.length - 1]} vs ${ - splitParam[splitParam.length - 1] - }`, - value: getCustomPlotId(metric, param) - } - }) + const plotsItems = plots.map(plot => + plot.type === CustomPlotType.CHECKPOINT + ? getCheckpointPlotItem(plot.metric) + : getMetricVsParamPlotItem(plot.metric, plot.param) + ) return quickPickManyValues(plotsItems, quickPickOptions) } +export const pickCustomPlotType = (): Thenable => { + return quickPickValue( + [ + { + description: + 'A linear plot that compares a chosen metric and param with current experiments.', + label: 'Metric Vs Param', + value: CustomPlotType.METRIC_VS_PARAM + }, + { + description: + 'A linear plot that shows how a chosen metric changes over selected experiments.', + label: 'Trend', + value: CustomPlotType.CHECKPOINT + } + ], + { + title: Title.SELECT_PLOT_TYPE_CUSTOM_PLOT + } + ) +} + const getTypeColumnLikes = (columns: Column[], columnType: ColumnType) => columns .filter(({ type }) => type === columnType) @@ -68,3 +115,21 @@ export const pickMetricAndParam = async (columns: Column[]) => { } return { metric: metric.path, param: param.path } } + +export const pickMetric = async (columns: Column[]) => { + const metricColumnLikes = getTypeColumnLikes(columns, ColumnType.METRICS) + + if (!definedAndNonEmpty(metricColumnLikes)) { + return Toast.showError('There are no metrics to select from.') + } + + const metric = await pickFromColumnLikes(metricColumnLikes, { + title: Title.SELECT_METRIC_CUSTOM_PLOT + }) + + if (!metric) { + return + } + + return metric.path +} diff --git a/extension/src/plots/webview/contract.ts b/extension/src/plots/webview/contract.ts index 2bc8149aca..ebb1decccb 100644 --- a/extension/src/plots/webview/contract.ts +++ b/extension/src/plots/webview/contract.ts @@ -72,6 +72,17 @@ export interface PlotsComparisonData { revisions: Revision[] } +export enum CustomPlotType { + CHECKPOINT = 'checkpoint', + METRIC_VS_PARAM = 'metricVsParam' +} + +export type MetricVsParamPlotValues = { + expName: string + metric: number + param: number +} + export type CheckpointPlotValues = { group: string iteration: number @@ -82,36 +93,39 @@ export type ColorScale = { domain: string[]; range: Color[] } export type CheckpointPlot = { id: string + metric: string values: CheckpointPlotValues + type: CustomPlotType.CHECKPOINT } -export type CustomPlotValues = { - expName: string - metric: number - param: number +export type CheckpointPlotData = CheckpointPlot & { yTitle: string } + +export type CheckpointPlotsData = { + plots: CheckpointPlotData[] + colors: ColorScale + nbItemsPerRow: number + height: number | undefined + selectedMetrics?: string[] } -export type CustomPlotData = { +export type MetricVsParamPlot = { id: string - values: CustomPlotValues[] + values: MetricVsParamPlotValues[] metric: string param: string + type: CustomPlotType.METRIC_VS_PARAM } -export type CustomPlotsData = { - plots: CustomPlotData[] - nbItemsPerRow: number - height: number | undefined -} +export type MetricVsParamPlotData = MetricVsParamPlot & { yTitle: string } -export type CheckpointPlotData = CheckpointPlot & { title: string } +export type CustomPlot = MetricVsParamPlot | CheckpointPlot +export type CustomPlotData = MetricVsParamPlotData | CheckpointPlotData -export type CheckpointPlotsData = { - plots: CheckpointPlotData[] - colors: ColorScale +export type CustomPlotsData = { + plots: CustomPlotData[] nbItemsPerRow: number height: number | undefined - selectedMetrics?: string[] + colors: ColorScale | undefined } export enum PlotsType { diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 1a4373ce68..ff8e9c85f8 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -2,6 +2,7 @@ import isEmpty from 'lodash.isempty' import { ComparisonPlot, ComparisonRevisionData, + CustomPlotType, PlotsData as TPlotsData, Revision, Section, @@ -21,11 +22,21 @@ import { PlotsModel } from '../model' import { PathsModel } from '../paths/model' import { BaseWebview } from '../../webview' import { getModifiedTime } from '../../fileSystem' -import { pickCustomPlots, pickMetricAndParam } from '../model/quickPick' +import { + pickCustomPlots, + pickCustomPlotType, + pickMetric, + pickMetricAndParam +} from '../model/quickPick' import { Title } from '../../vscode/title' import { ColumnType } from '../../experiments/webview/contract' import { FILE_SEPARATOR } from '../../experiments/columns/paths' import { reorderObjectList } from '../../util/array' +import { isCheckpointPlot } from '../model/collect' +import { + CustomPlotsOrderValue, + isCustomPlotOrderCheckpointValue +} from '../model/custom' export class WebviewMessages { private readonly paths: PathsModel @@ -179,7 +190,9 @@ export class WebviewMessages { this.sendCheckpointPlotsAndEvent(EventName.VIEWS_REORDER_PLOTS_METRICS) } - private async addCustomPlot() { + private async getMetricOrParamPlot(): Promise< + CustomPlotsOrderValue | undefined + > { const metricAndParam = await pickMetricAndParam( this.experiments.getColumnTerminalNodes() ) @@ -188,18 +201,68 @@ export class WebviewMessages { return } - const plotAlreadyExists = this.plots - .getCustomPlotsOrder() - .some( - ({ param, metric }) => - param === metricAndParam.param && metric === metricAndParam.metric + const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { + if (isCustomPlotOrderCheckpointValue(value)) { + return + } + return ( + value.param === metricAndParam.param && + value.metric === metricAndParam.metric ) + }) + + if (plotAlreadyExists) { + return Toast.showError('Custom plot already exists.') + } + + return { + ...metricAndParam, + type: CustomPlotType.METRIC_VS_PARAM + } + } + + private async getCheckpointPlot(): Promise< + CustomPlotsOrderValue | undefined + > { + const metric = await pickMetric(this.experiments.getColumnTerminalNodes()) + + if (!metric) { + return + } + + const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { + if (isCustomPlotOrderCheckpointValue(value)) { + return value.metric === metric + } + }) if (plotAlreadyExists) { return Toast.showError('Custom plot already exists.') } - this.plots.addCustomPlot(metricAndParam) + return { + metric, + type: CustomPlotType.CHECKPOINT + } + } + + private async addCustomPlot() { + const plotType = await pickCustomPlotType() + + if (!plotType) { + return + } + + const plot = await (plotType === CustomPlotType.CHECKPOINT + ? this.getCheckpointPlot() + : this.getMetricOrParamPlot()) + + if (!plot) { + return + } + + this.plots.addCustomPlot(plot) + this.sendCustomPlots() sendTelemetryEvent( EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED, @@ -238,11 +301,21 @@ export class WebviewMessages { const buildMetricOrParamPath = (type: string, path: string) => type + FILE_SEPARATOR + path - const newOrder = reorderObjectList(plotIds, customPlots, 'id').map( - ({ metric, param }) => ({ - metric: buildMetricOrParamPath(ColumnType.METRICS, metric), - param: buildMetricOrParamPath(ColumnType.PARAMS, param) - }) + const newOrder: CustomPlotsOrderValue[] = reorderObjectList( + plotIds, + customPlots, + 'id' + ).map(plot => + isCheckpointPlot(plot) + ? { + metric: buildMetricOrParamPath(ColumnType.METRICS, plot.title), + type: CustomPlotType.CHECKPOINT + } + : { + metric: buildMetricOrParamPath(ColumnType.METRICS, plot.metric), + param: buildMetricOrParamPath(ColumnType.PARAMS, plot.param), + type: CustomPlotType.METRIC_VS_PARAM + } ) this.plots.setCustomPlotsOrder(newOrder) this.sendCustomPlots() diff --git a/extension/src/vscode/title.ts b/extension/src/vscode/title.ts index 39be6d0c11..eecb9ff650 100644 --- a/extension/src/vscode/title.ts +++ b/extension/src/vscode/title.ts @@ -25,6 +25,7 @@ export enum Title { SELECT_PARAM_OR_METRIC_SORT = 'Select a Param or Metric to Sort by', SELECT_METRIC_CUSTOM_PLOT = 'Select a Metric to Create a Custom Plot', SELECT_PARAM_CUSTOM_PLOT = 'Select a Param to Create a Custom Plot', + SELECT_PLOT_TYPE_CUSTOM_PLOT = 'Select a Custom Plot Type', SELECT_CUSTOM_PLOTS_TO_REMOVE = 'Select Custom Plot(s) to Remove', SELECT_PARAM_TO_MODIFY = 'Select Param(s) to Modify', SELECT_PLOTS = 'Select Plots to Display', diff --git a/webview/src/plots/components/Plots.tsx b/webview/src/plots/components/Plots.tsx index 57db03b854..adced4ab44 100644 --- a/webview/src/plots/components/Plots.tsx +++ b/webview/src/plots/components/Plots.tsx @@ -2,7 +2,6 @@ import React, { createRef, useLayoutEffect } from 'react' import { useSelector, useDispatch } from 'react-redux' import { AddPlots, Welcome } from './GetStarted' import { ZoomedInPlot } from './ZoomedInPlot' -import { CheckpointPlotsWrapper } from './checkpointPlots/CheckpointPlotsWrapper' import { CustomPlotsWrapper } from './customPlots/CustomPlotsWrapper' import { TemplatePlotsWrapper } from './templatePlots/TemplatePlotsWrapper' import { ComparisonTableWrapper } from './comparisonTable/ComparisonTableWrapper' @@ -20,9 +19,6 @@ const PlotsContent = () => { const { hasData, hasPlots, hasUnselectedPlots, zoomedInPlot } = useSelector( (state: PlotsState) => state.webview ) - const hasCheckpointData = useSelector( - (state: PlotsState) => state.checkpoint.hasData - ) const hasComparisonData = useSelector( (state: PlotsState) => state.comparison.hasData ) @@ -49,7 +45,7 @@ const PlotsContent = () => { return Loading Plots... } - if (!hasCheckpointData && !hasComparisonData && !hasTemplateData) { + if (!hasComparisonData && !hasTemplateData) { return ( } @@ -64,7 +60,6 @@ const PlotsContent = () => { - {zoomedInPlot?.plot && ( diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index 07bd13a14a..b31e6f1811 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -1,7 +1,13 @@ -import { Section } from 'dvc/src/plots/webview/contract' +import { + CheckpointPlotData, + ColorScale, + CustomPlotData, + CustomPlotType, + Section +} from 'dvc/src/plots/webview/contract' import React, { useMemo, useEffect, useState } from 'react' import { useSelector } from 'react-redux' -import { createSpec } from './util' +import { createMetricVsParamSpec, createCheckpointSpec } from './util' import { changeDisabledDragIds, changeSize } from './customPlotsSlice' import { ZoomablePlot } from '../ZoomablePlot' import styles from '../styles.module.scss' @@ -13,26 +19,43 @@ interface CustomPlotProps { id: string } +const isCheckpointPlot = (plot: CustomPlotData): plot is CheckpointPlotData => { + return plot.type === CustomPlotType.CHECKPOINT +} + +const createCustomPlotSpec = ( + plot: CustomPlotData | undefined, + colors: ColorScale | undefined +) => { + if (!plot) { + return {} + } + // TBD were forced to use this type of "if or" statement mutliple times throughout the custom code + // There's probably a better way to do this + if (isCheckpointPlot(plot)) { + return colors ? createCheckpointSpec(plot.yTitle, colors) : {} + } + return createMetricVsParamSpec(plot.yTitle, plot.param) +} + export const CustomPlot: React.FC = ({ id }) => { const plotSnapshot = useSelector( (state: PlotsState) => state.custom.plotsSnapshots[id] ) + const [plot, setPlot] = useState(plotDataStore[Section.CUSTOM_PLOTS][id]) - const nbItemsPerRow = useSelector( - (state: PlotsState) => state.custom.nbItemsPerRow + const { nbItemsPerRow, colors } = useSelector( + (state: PlotsState) => state.custom ) - const spec = useMemo(() => { - if (plot) { - return createSpec(plot.metric, plot.param) - } - }, [plot]) + return createCustomPlotSpec(plot, colors) + }, [plot, colors]) useEffect(() => { setPlot(plotDataStore[Section.CUSTOM_PLOTS][id]) }, [plotSnapshot, id]) - if (!plot || !spec) { + if (!plot) { return null } diff --git a/webview/src/plots/components/customPlots/customPlotsSlice.ts b/webview/src/plots/components/customPlots/customPlotsSlice.ts index 95d5082bc9..5e2cb2da38 100644 --- a/webview/src/plots/components/customPlots/customPlotsSlice.ts +++ b/webview/src/plots/components/customPlots/customPlotsSlice.ts @@ -17,6 +17,7 @@ export interface CustomPlotsState extends Omit { } export const customPlotsInitialState: CustomPlotsState = { + colors: { domain: [], range: [] }, disabledDragPlotIds: [], hasData: false, height: DEFAULT_HEIGHT[Section.CUSTOM_PLOTS], @@ -43,13 +44,14 @@ export const customPlotsSlice = createSlice({ if (!action.payload) { return customPlotsInitialState } - const { plots, ...statePayload } = action.payload + const { plots, colors, ...statePayload } = action.payload const plotsIds = plots?.map(plot => plot.id) || [] const snapShots = addPlotsWithSnapshots(plots, Section.CUSTOM_PLOTS) removePlots(plotsIds, Section.CUSTOM_PLOTS) return { ...state, ...statePayload, + colors: colors || { domain: [], range: [] }, hasData: !!action.payload, plotsIds: plots?.map(plot => plot.id) || [], plotsSnapshots: snapShots diff --git a/webview/src/plots/components/customPlots/util.ts b/webview/src/plots/components/customPlots/util.ts index bbb7cfbd49..92deef218b 100644 --- a/webview/src/plots/components/customPlots/util.ts +++ b/webview/src/plots/components/customPlots/util.ts @@ -1,6 +1,98 @@ import { VisualizationSpec } from 'react-vega' +import { ColorScale } from 'dvc/src/plots/webview/contract' -export const createSpec = (metric: string, param: string) => +export const createCheckpointSpec = ( + title: string, + scale?: ColorScale +): VisualizationSpec => + ({ + $schema: 'https://vega.github.io/schema/vega-lite/v5.json', + data: { name: 'values' }, + encoding: { + color: { + field: 'group', + legend: { disable: true }, + scale, + title: 'rev', + type: 'nominal' + }, + x: { + axis: { format: '0d', tickMinStep: 1 }, + field: 'iteration', + title: 'iteration', + type: 'quantitative' + }, + y: { + field: 'y', + scale: { zero: false }, + title, + type: 'quantitative' + } + }, + height: 'container', + layer: [ + { + layer: [ + { mark: { type: 'line' } }, + { + mark: { type: 'point' }, + transform: [ + { + filter: { empty: false, param: 'hover' } + } + ] + } + ] + }, + { + encoding: { + opacity: { value: 0 }, + tooltip: [ + { field: 'group', title: 'name' }, + { + field: 'y', + title: title.slice(Math.max(0, title.indexOf(':') + 1)), + type: 'quantitative' + } + ] + }, + mark: { type: 'rule' }, + params: [ + { + name: 'hover', + select: { + clear: 'mouseout', + fields: ['iteration', 'y'], + nearest: true, + on: 'mouseover', + type: 'point' + } + } + ] + }, + { + encoding: { + color: { field: 'group', scale }, + x: { aggregate: 'max', field: 'iteration', type: 'quantitative' }, + y: { + aggregate: { argmax: 'iteration' }, + field: 'y', + type: 'quantitative' + } + }, + mark: { stroke: null, type: 'circle' } + } + ], + transform: [ + { + as: 'y', + calculate: "format(datum['y'],'.5f')" + } + ], + width: 'container' + } as VisualizationSpec) +// TBD rename to title? +export const createMetricVsParamSpec = (metric: string, param: string) => ({ $schema: 'https://vega.github.io/schema/vega-lite/v5.json', data: { name: 'values' }, From 6dc8813a8b0007894caecfba851ae1030783c54c Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 6 Mar 2023 10:03:55 -0600 Subject: [PATCH 02/40] fix typo --- extension/src/plots/webview/messages.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index ff8e9c85f8..12faeeaa2b 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -308,7 +308,7 @@ export class WebviewMessages { ).map(plot => isCheckpointPlot(plot) ? { - metric: buildMetricOrParamPath(ColumnType.METRICS, plot.title), + metric: buildMetricOrParamPath(ColumnType.METRICS, plot.metric), type: CustomPlotType.CHECKPOINT } : { From 88fe26d61d7f0ed4d45088fc8c6536a41c735c19 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 6 Mar 2023 10:40:03 -0600 Subject: [PATCH 03/40] fix linting --- extension/src/plots/model/collect.test.ts | 129 +----------------- extension/src/plots/model/collect.ts | 63 +-------- extension/src/plots/model/index.ts | 13 -- extension/src/plots/model/quickPick.test.ts | 12 +- extension/src/plots/webview/messages.ts | 7 - .../fixtures/expShow/base/checkpointPlots.ts | 33 +++-- .../test/fixtures/expShow/base/customPlots.ts | 16 ++- extension/src/test/suite/plots/index.test.ts | 57 +------- extension/src/test/suite/plots/util.ts | 2 +- webview/src/plots/components/App.test.tsx | 55 ++++---- webview/src/plots/components/App.tsx | 9 -- .../checkpointPlots/CheckpointPlot.tsx | 59 -------- .../checkpointPlots/CheckpointPlots.tsx | 117 ---------------- .../CheckpointPlotsWrapper.tsx | 49 ------- .../checkpointPlots/checkpointPlotsSlice.ts | 67 --------- .../plots/components/checkpointPlots/util.ts | 93 ------------- webview/src/plots/hooks/useGetPlot.ts | 12 +- webview/src/plots/store.ts | 2 - webview/src/stories/Plots.stories.tsx | 13 -- 19 files changed, 86 insertions(+), 722 deletions(-) delete mode 100644 webview/src/plots/components/checkpointPlots/CheckpointPlot.tsx delete mode 100644 webview/src/plots/components/checkpointPlots/CheckpointPlots.tsx delete mode 100644 webview/src/plots/components/checkpointPlots/CheckpointPlotsWrapper.tsx delete mode 100644 webview/src/plots/components/checkpointPlots/checkpointPlotsSlice.ts delete mode 100644 webview/src/plots/components/checkpointPlots/util.ts diff --git a/extension/src/plots/model/collect.test.ts b/extension/src/plots/model/collect.test.ts index 0b81f32648..72425d3d2e 100644 --- a/extension/src/plots/model/collect.test.ts +++ b/extension/src/plots/model/collect.test.ts @@ -5,7 +5,6 @@ import { collectData, collectCheckpointPlotsData, collectTemplates, - collectMetricOrder, collectOverrideRevisionDetails, collectCustomPlotsData } from './collect' @@ -20,7 +19,7 @@ import { EXPERIMENT_WORKSPACE_ID } from '../../cli/dvc/contract' import { definedAndNonEmpty, sameContents } from '../../util/array' -import { TemplatePlot } from '../webview/contract' +import { CustomPlotType, TemplatePlot } from '../webview/contract' import { getCLICommitId } from '../../test/fixtures/plotsDiff/util' import { SelectedExperimentWithColor } from '../../experiments/model' import { Experiment } from '../../experiments/webview/contract' @@ -35,13 +34,16 @@ describe('collectCustomPlotsData', () => { [ { metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout' + param: 'params:params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM }, { metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:epochs' + param: 'params:params.yaml:epochs', + type: CustomPlotType.METRIC_VS_PARAM } ], + {}, [ { id: '12345', @@ -137,125 +139,6 @@ describe('collectCheckpointPlotsData', () => { }) }) -describe('collectMetricOrder', () => { - it('should return an empty array if there is no checkpoints data', () => { - const metricOrder = collectMetricOrder( - undefined, - ['metric:A', 'metric:B'], - [] - ) - expect(metricOrder).toStrictEqual([]) - }) - - it('should return an empty array if the checkpoints data is an empty array', () => { - const metricOrder = collectMetricOrder([], ['metric:A', 'metric:B'], []) - expect(metricOrder).toStrictEqual([]) - }) - - it('should maintain the existing order if all metrics are selected', () => { - const expectedOrder = [ - 'metric:F', - 'metric:A', - 'metric:B', - 'metric:E', - 'metric:D', - 'metric:C' - ] - - const metricOrder = collectMetricOrder( - [ - { id: 'metric:A', values: [] }, - { id: 'metric:B', values: [] }, - { id: 'metric:C', values: [] }, - { id: 'metric:D', values: [] }, - { id: 'metric:E', values: [] }, - { id: 'metric:F', values: [] } - ], - expectedOrder, - expectedOrder - ) - expect(metricOrder).toStrictEqual(expectedOrder) - }) - - it('should push unselected metrics to the end', () => { - const existingOrder = [ - 'metric:F', - 'metric:A', - 'metric:B', - 'metric:E', - 'metric:D', - 'metric:C' - ] - - const metricOrder = collectMetricOrder( - [ - { id: 'metric:A', values: [] }, - { id: 'metric:B', values: [] }, - { id: 'metric:C', values: [] }, - { id: 'metric:D', values: [] }, - { id: 'metric:E', values: [] }, - { id: 'metric:F', values: [] } - ], - existingOrder, - existingOrder.filter(metric => !['metric:A', 'metric:B'].includes(metric)) - ) - expect(metricOrder).toStrictEqual([ - 'metric:F', - 'metric:E', - 'metric:D', - 'metric:C', - 'metric:A', - 'metric:B' - ]) - }) - - it('should add new metrics in the given order', () => { - const metricOrder = collectMetricOrder( - [ - { id: 'metric:C', values: [] }, - { id: 'metric:D', values: [] }, - { id: 'metric:A', values: [] }, - { id: 'metric:B', values: [] }, - { id: 'metric:E', values: [] }, - { id: 'metric:F', values: [] } - ], - ['metric:B', 'metric:A'], - ['metric:B', 'metric:A'] - ) - expect(metricOrder).toStrictEqual([ - 'metric:B', - 'metric:A', - 'metric:C', - 'metric:D', - 'metric:E', - 'metric:F' - ]) - }) - - it('should give selected metrics precedence', () => { - const metricOrder = collectMetricOrder( - [ - { id: 'metric:C', values: [] }, - { id: 'metric:D', values: [] }, - { id: 'metric:A', values: [] }, - { id: 'metric:B', values: [] }, - { id: 'metric:E', values: [] }, - { id: 'metric:F', values: [] } - ], - ['metric:B', 'metric:A'], - ['metric:B', 'metric:A', 'metric:F'] - ) - expect(metricOrder).toStrictEqual([ - 'metric:B', - 'metric:A', - 'metric:F', - 'metric:C', - 'metric:D', - 'metric:E' - ]) - }) -}) - describe('collectData', () => { it('should return the expected output from the test fixture', () => { const mapping = { diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 53cb005736..0571521e1b 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -48,11 +48,7 @@ import { import { addToMapArray } from '../../util/map' import { TemplateOrder } from '../paths/collect' import { extendVegaSpec, isMultiViewPlot } from '../vega/util' -import { - definedAndNonEmpty, - reorderObjectList, - splitMatchedOrdered -} from '../../util/array' +import { definedAndNonEmpty, reorderObjectList } from '../../util/array' import { shortenForLabel } from '../../util/string' import { getDvcDataVersionInfo, @@ -360,63 +356,6 @@ export const collectCustomPlotsData = ( .filter(Boolean) } -type MetricOrderAccumulator = { - newOrder: string[] - uncollectedMetrics: string[] - remainingSelectedMetrics: string[] -} - -const collectExistingOrder = ( - acc: MetricOrderAccumulator, - existingMetricOrder: string[] -) => { - for (const metric of existingMetricOrder) { - const uncollectedIndex = acc.uncollectedMetrics.indexOf(metric) - const remainingIndex = acc.remainingSelectedMetrics.indexOf(metric) - if (uncollectedIndex === -1 || remainingIndex === -1) { - continue - } - acc.uncollectedMetrics.splice(uncollectedIndex, 1) - acc.remainingSelectedMetrics.splice(remainingIndex, 1) - acc.newOrder.push(metric) - } -} - -const collectRemainingSelected = (acc: MetricOrderAccumulator) => { - const [newOrder, uncollectedMetrics] = splitMatchedOrdered( - acc.uncollectedMetrics, - acc.remainingSelectedMetrics - ) - - acc.newOrder.push(...newOrder) - acc.uncollectedMetrics = uncollectedMetrics -} - -export const collectMetricOrder = ( - checkpointPlotData: CheckpointPlot[] | undefined, - existingMetricOrder: string[], - selectedMetrics: string[] = [] -): string[] => { - if (!definedAndNonEmpty(checkpointPlotData)) { - return [] - } - - const acc: MetricOrderAccumulator = { - newOrder: [], - remainingSelectedMetrics: [...selectedMetrics], - uncollectedMetrics: checkpointPlotData.map(({ id }) => id) - } - - if (!definedAndNonEmpty(acc.remainingSelectedMetrics)) { - return acc.uncollectedMetrics - } - - collectExistingOrder(acc, existingMetricOrder) - collectRemainingSelected(acc) - - return [...acc.newOrder, ...acc.uncollectedMetrics] -} - type RevisionPathData = { [path: string]: Record[] } export type RevisionData = { diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index aaf6f9d747..506c2f4fbd 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -4,7 +4,6 @@ import isEqual from 'lodash.isequal' import { collectCheckpointPlotsData, collectData, - collectMetricOrder, collectSelectedTemplatePlots, collectTemplates, ComparisonData, @@ -119,8 +118,6 @@ export class PlotsModel extends ModelWithPersistence { this.checkpointPlots = checkpointPlots - this.setMetricOrder() - this.recreateCustomPlots(data) return this.removeStaleData() @@ -401,7 +398,6 @@ export class PlotsModel extends ModelWithPersistence { public setSelectedMetrics(selectedMetrics: string[]) { this.selectedMetrics = selectedMetrics - this.setMetricOrder() this.persist( PersistenceKey.PLOT_SELECTED_METRICS, this.getSelectedMetrics() @@ -412,15 +408,6 @@ export class PlotsModel extends ModelWithPersistence { return this.selectedMetrics } - public setMetricOrder(metricOrder?: string[]) { - this.metricOrder = collectMetricOrder( - this.checkpointPlots, - metricOrder || this.metricOrder, - this.selectedMetrics - ) - this.persist(PersistenceKey.PLOT_METRIC_ORDER, this.metricOrder) - } - public setNbItemsPerRow(section: Section, nbItemsPerRow: number) { this.nbItemsPerRow[section] = nbItemsPerRow this.persist(PersistenceKey.PLOT_NB_ITEMS_PER_ROW, this.nbItemsPerRow) diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index 363b3503a8..3e195e9117 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -1,9 +1,10 @@ -import { CustomPlotsOrderValue } from '.' +import { CustomPlotsOrderValue } from './custom' import { pickCustomPlots, pickMetricAndParam } from './quickPick' import { quickPickManyValues, quickPickValue } from '../../vscode/quickPick' import { Title } from '../../vscode/title' import { Toast } from '../../vscode/toast' import { ColumnType } from '../../experiments/webview/contract' +import { CustomPlotType } from '../webview/contract' jest.mock('../../vscode/quickPick') jest.mock('../../vscode/toast') @@ -35,15 +36,18 @@ describe('pickCustomPlots', () => { const mockedExperiments = [ { metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout' + param: 'params:params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM }, { metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:epochs' + param: 'params:params.yaml:epochs', + type: CustomPlotType.METRIC_VS_PARAM }, { metric: 'metrics:summary.json:learning_rate', - param: 'param:summary.json:process.threshold' + param: 'param:summary.json:process.threshold', + type: CustomPlotType.METRIC_VS_PARAM } ] as CustomPlotsOrderValue[] diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 12faeeaa2b..b7264fd60a 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -105,8 +105,6 @@ export class WebviewMessages { return this.setComparisonRowsOrder(message.payload) case MessageFromWebviewType.REORDER_PLOTS_TEMPLATES: return this.setTemplateOrder(message.payload) - case MessageFromWebviewType.REORDER_PLOTS_METRICS: - return this.setMetricOrder(message.payload) case MessageFromWebviewType.REORDER_PLOTS_CUSTOM: return this.setCustomPlotsOrder(message.payload) case MessageFromWebviewType.SELECT_PLOTS: @@ -185,11 +183,6 @@ export class WebviewMessages { ) } - private setMetricOrder(order: string[]) { - this.plots.setMetricOrder(order) - this.sendCheckpointPlotsAndEvent(EventName.VIEWS_REORDER_PLOTS_METRICS) - } - private async getMetricOrParamPlot(): Promise< CustomPlotsOrderValue | undefined > { diff --git a/extension/src/test/fixtures/expShow/base/checkpointPlots.ts b/extension/src/test/fixtures/expShow/base/checkpointPlots.ts index 4b4f620e02..c0d146e558 100644 --- a/extension/src/test/fixtures/expShow/base/checkpointPlots.ts +++ b/extension/src/test/fixtures/expShow/base/checkpointPlots.ts @@ -1,6 +1,7 @@ import { copyOriginalColors } from '../../../../experiments/model/status/colors' import { CheckpointPlotsData, + CustomPlotType, PlotNumberOfItemsPerRow } from '../../../../plots/webview/contract' @@ -13,8 +14,8 @@ const data: CheckpointPlotsData = { }, plots: [ { - id: 'summary.json:loss', - title: 'summary.json:loss', + id: 'custom-summary.json:loss', + metric: 'summary.json:loss', values: [ { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, @@ -28,11 +29,13 @@ const data: CheckpointPlotsData = { { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } - ] + ], + type: CustomPlotType.CHECKPOINT, + yTitle: 'summary.json:loss' }, { - id: 'summary.json:accuracy', - title: 'summary.json:accuracy', + id: 'custom-summary.json:accuracy', + metric: 'summary.json:accuracy', values: [ { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, @@ -46,11 +49,13 @@ const data: CheckpointPlotsData = { { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } - ] + ], + type: CustomPlotType.CHECKPOINT, + yTitle: 'summary.json:accuracy' }, { - id: 'summary.json:val_loss', - title: 'summary.json:val_loss', + id: 'custom-summary.json:val_loss', + metric: 'summary.json:val_loss', values: [ { group: 'exp-83425', iteration: 1, y: 1.9391471147537231 }, { group: 'exp-83425', iteration: 2, y: 1.8825950622558594 }, @@ -64,11 +69,13 @@ const data: CheckpointPlotsData = { { group: 'exp-e7a67', iteration: 1, y: 1.9979370832443237 }, { group: 'exp-e7a67', iteration: 2, y: 1.9979370832443237 }, { group: 'exp-e7a67', iteration: 3, y: 1.9979370832443237 } - ] + ], + type: CustomPlotType.CHECKPOINT, + yTitle: 'summary.json:val_loss' }, { - id: 'summary.json:val_accuracy', - title: 'summary.json:val_accuracy', + id: 'custom-summary.json:val_accuracy', + metric: 'summary.json:val_accuracy', values: [ { group: 'exp-83425', iteration: 1, y: 0.49399998784065247 }, { group: 'exp-83425', iteration: 2, y: 0.5550000071525574 }, @@ -82,7 +89,9 @@ const data: CheckpointPlotsData = { { group: 'exp-e7a67', iteration: 1, y: 0.4277999997138977 }, { group: 'exp-e7a67', iteration: 2, y: 0.4277999997138977 }, { group: 'exp-e7a67', iteration: 3, y: 0.4277999997138977 } - ] + ], + type: CustomPlotType.CHECKPOINT, + yTitle: 'summary.json:val_accuracy' } ], selectedMetrics: [ diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index 67bcfa8d62..3f1c5e384f 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -1,14 +1,23 @@ +import { copyOriginalColors } from '../../../../experiments/model/status/colors' import { CustomPlotsData, + CustomPlotType, PlotNumberOfItemsPerRow } from '../../../../plots/webview/contract' +const colors = copyOriginalColors() + const data: CustomPlotsData = { + colors: { + domain: ['exp-e7a67', 'test-branch', 'exp-83425'], + range: [colors[2], colors[3], colors[4]] + }, plots: [ { id: 'custom-metrics:summary.json:loss-params:params.yaml:dropout', metric: 'summary.json:loss', param: 'params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM, values: [ { expName: 'exp-e7a67', @@ -25,12 +34,14 @@ const data: CustomPlotsData = { metric: 2.298503875732422, param: 0.32 } - ] + ], + yTitle: 'summary.json:loss' }, { id: 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', metric: 'summary.json:accuracy', param: 'params.yaml:epochs', + type: CustomPlotType.METRIC_VS_PARAM, values: [ { expName: 'exp-e7a67', @@ -47,7 +58,8 @@ const data: CustomPlotsData = { metric: 0.6768440509033, param: 20 } - ] + ], + yTitle: 'summary.json:accuracy' } ], nbItemsPerRow: PlotNumberOfItemsPerRow.TWO, diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index 8c4e45683d..b7c4110a7b 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -28,7 +28,8 @@ import { PlotNumberOfItemsPerRow, Section, TemplatePlotGroup, - TemplatePlotsData + TemplatePlotsData, + CustomPlotType } from '../../../plots/webview/contract' import { TEMP_PLOTS_DIR } from '../../../cli/dvc/constants' import { WEBVIEW_TEST_TIMEOUT } from '../timeouts' @@ -465,55 +466,6 @@ suite('Plots Test Suite', () => { ) }).timeout(WEBVIEW_TEST_TIMEOUT) - it('should handle a metric reordered message from the webview', async () => { - const { plots, plotsModel, messageSpy } = await buildPlots( - disposable, - plotsDiffFixture - ) - - const webview = await plots.showWebview() - - const mockSendTelemetryEvent = stub(Telemetry, 'sendTelemetryEvent') - const mockMessageReceived = getMessageReceivedEmitter(webview) - - const mockSetMetricOrder = spy(plotsModel, 'setMetricOrder') - const mockMetricOrder = [ - 'summary.json:loss', - 'summary.json:accuracy', - 'summary.json:val_loss', - 'summary.json:val_accuracy' - ] - - messageSpy.resetHistory() - mockMessageReceived.fire({ - payload: mockMetricOrder, - type: MessageFromWebviewType.REORDER_PLOTS_METRICS - }) - - expect(mockSetMetricOrder).to.be.calledOnce - expect(mockSetMetricOrder).to.be.calledWithExactly(mockMetricOrder) - expect(messageSpy).to.be.calledOnce - expect( - messageSpy, - "should update the webview's checkpoint plot order state" - ).to.be.calledWithExactly({ - checkpoint: { - ...checkpointPlotsFixture, - plots: reorderObjectList( - mockMetricOrder, - checkpointPlotsFixture.plots, - 'title' - ) - } - }) - expect(mockSendTelemetryEvent).to.be.calledOnce - expect(mockSendTelemetryEvent).to.be.calledWithExactly( - EventName.VIEWS_REORDER_PLOTS_METRICS, - undefined, - undefined - ) - }).timeout(WEBVIEW_TEST_TIMEOUT) - it('should handle a custom plots reordered message from the webview', async () => { const { plots, plotsModel, messageSpy } = await buildPlots( disposable, @@ -729,7 +681,7 @@ suite('Plots Test Suite', () => { const expectedPlotsData: TPlotsData = { checkpoint: checkpointPlotsFixture, comparison: comparisonPlotsFixture, - custom: { height: undefined, nbItemsPerRow: 2, plots: [] }, + custom: customPlotsFixture, hasPlots: true, hasUnselectedPlots: false, sectionCollapsed: DEFAULT_SECTION_COLLAPSED, @@ -925,7 +877,8 @@ suite('Plots Test Suite', () => { stub(plotsModel, 'getCustomPlotsOrder').returns([ { metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout' + param: 'params:params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM } ]) diff --git a/extension/src/test/suite/plots/util.ts b/extension/src/test/suite/plots/util.ts index 007783f886..00ebb17230 100644 --- a/extension/src/test/suite/plots/util.ts +++ b/extension/src/test/suite/plots/util.ts @@ -143,7 +143,7 @@ export const getExpectedCheckpointPlotsData = ( nbItemsPerRow, plots: plots.map(plot => ({ id: plot.id, - title: plot.title, + title: plot.yTitle, values: plot.values.filter(values => domain.includes(values.group)) })), selectedMetrics diff --git a/webview/src/plots/components/App.test.tsx b/webview/src/plots/components/App.test.tsx index 8e917e8580..e4f0bb3eec 100644 --- a/webview/src/plots/components/App.test.tsx +++ b/webview/src/plots/components/App.test.tsx @@ -28,7 +28,8 @@ import { Revision, Section, TemplatePlotGroup, - TemplatePlotsData + TemplatePlotsData, + CustomPlotType } from 'dvc/src/plots/webview/contract' import { MessageFromWebviewType, @@ -73,18 +74,16 @@ jest.mock('../../shared/components/dragDrop/currentTarget', () => { jest.mock('../../shared/api') -jest.mock('./checkpointPlots/util', () => ({ - createSpec: () => ({ +jest.mock('./customPlots/util', () => ({ + createCheckpointSpec: () => ({ $schema: 'https://vega.github.io/schema/vega-lite/v5.json', encoding: {}, height: 100, layer: [], transform: [], width: 100 - }) -})) -jest.mock('./customPlots/util', () => ({ - createSpec: () => ({ + }), + createMetricVsParamSpec: () => ({ $schema: 'https://vega.github.io/schema/vega-lite/v5.json', encoding: {}, height: 100, @@ -1521,21 +1520,23 @@ describe('App', () => { }) describe('Virtualization', () => { - const createCheckpointPlots = (nbOfPlots: number) => { + const createCheckpointPlots = (nbOfPlots: number): CheckpointPlotsData => { const plots = [] for (let i = 0; i < nbOfPlots; i++) { const id = `plot-${i}` plots.push({ id, - title: id, - values: [] + metric: '', + type: CustomPlotType.CHECKPOINT, + values: [], + yTitle: id }) } return { ...checkpointPlotsFixture, plots, selectedMetrics: plots.map(plot => plot.id) - } + } as CheckpointPlotsData } const resizeScreen = (width: number, store: typeof plotsStore) => { @@ -1638,14 +1639,14 @@ describe('App', () => { let plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].title) + expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) expect(plots.length).toBe(OVERSCAN_ROW_COUNT + 1) resizeScreen(5453, store) plots = screen.getAllByTestId(/^plot-/) - expect(plots[3].id).toBe(checkpoint.plots[3].title) + expect(plots[3].id).toBe(checkpoint.plots[3].yTitle) expect(plots.length).toBe(OVERSCAN_ROW_COUNT + 1) }) @@ -1654,7 +1655,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[12].id).toBe(checkpoint.plots[12].title) + expect(plots[12].id).toBe(checkpoint.plots[12].yTitle) expect(plots.length).toBe(OVERSCAN_ROW_COUNT + 1) }) @@ -1663,7 +1664,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[14].id).toBe(checkpoint.plots[14].title) + expect(plots[14].id).toBe(checkpoint.plots[14].yTitle) expect(plots.length).toBe(1 + OVERSCAN_ROW_COUNT) // Only the first and the next lines defined by the overscan row count will be rendered }) @@ -1672,7 +1673,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].title) + expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) }) }) }) @@ -1735,14 +1736,14 @@ describe('App', () => { let plots = screen.getAllByTestId(/^plot-/) - expect(plots[20].id).toBe(checkpoint.plots[20].title) + expect(plots[20].id).toBe(checkpoint.plots[20].yTitle) expect(plots.length).toBe(checkpoint.plots.length) resizeScreen(6453, store) plots = screen.getAllByTestId(/^plot-/) - expect(plots[19].id).toBe(checkpoint.plots[19].title) + expect(plots[19].id).toBe(checkpoint.plots[19].yTitle) expect(plots.length).toBe(checkpoint.plots.length) }) @@ -1751,7 +1752,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[7].id).toBe(checkpoint.plots[7].title) + expect(plots[7].id).toBe(checkpoint.plots[7].yTitle) expect(plots.length).toBe(checkpoint.plots.length) }) @@ -1760,7 +1761,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[7].id).toBe(checkpoint.plots[7].title) + expect(plots[7].id).toBe(checkpoint.plots[7].yTitle) expect(plots.length).toBe(checkpoint.plots.length) }) @@ -1769,7 +1770,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].title) + expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) }) }) }) @@ -1832,14 +1833,14 @@ describe('App', () => { let plots = screen.getAllByTestId(/^plot-/) - expect(plots[7].id).toBe(checkpoint.plots[7].title) + expect(plots[7].id).toBe(checkpoint.plots[7].yTitle) expect(plots.length).toBe(checkpoint.plots.length) resizeScreen(5473, store) plots = screen.getAllByTestId(/^plot-/) - expect(plots[9].id).toBe(checkpoint.plots[9].title) + expect(plots[9].id).toBe(checkpoint.plots[9].yTitle) expect(plots.length).toBe(checkpoint.plots.length) }) @@ -1848,7 +1849,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[24].id).toBe(checkpoint.plots[24].title) + expect(plots[24].id).toBe(checkpoint.plots[24].yTitle) expect(plots.length).toBe(checkpoint.plots.length) }) @@ -1857,7 +1858,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[9].id).toBe(checkpoint.plots[9].title) + expect(plots[9].id).toBe(checkpoint.plots[9].yTitle) expect(plots.length).toBe(checkpoint.plots.length) }) @@ -1866,7 +1867,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[9].id).toBe(checkpoint.plots[9].title) + expect(plots[9].id).toBe(checkpoint.plots[9].yTitle) expect(plots.length).toBe(checkpoint.plots.length) }) @@ -1875,7 +1876,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].title) + expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) }) }) }) diff --git a/webview/src/plots/components/App.tsx b/webview/src/plots/components/App.tsx index 67e7ead963..409f10b27b 100644 --- a/webview/src/plots/components/App.tsx +++ b/webview/src/plots/components/App.tsx @@ -1,7 +1,6 @@ import React, { useCallback } from 'react' import { useDispatch } from 'react-redux' import { - CheckpointPlotsData, CustomPlotsData, PlotsComparisonData, PlotsData, @@ -13,10 +12,6 @@ import { } from 'dvc/src/plots/webview/contract' import { MessageToWebview } from 'dvc/src/webview/contract' import { Plots } from './Plots' -import { - setCollapsed as setCheckpointPlotsCollapsed, - update as updateCheckpointPlots -} from './checkpointPlots/checkpointPlotsSlice' import { setCollapsed as setCustomPlotsCollapsed, update as updateCustomPlots @@ -43,7 +38,6 @@ const dispatchCollapsedSections = ( dispatch: PlotsDispatch ) => { if (sections) { - dispatch(setCheckpointPlotsCollapsed(sections[Section.CHECKPOINT_PLOTS])) dispatch(setCustomPlotsCollapsed(sections[Section.CUSTOM_PLOTS])) dispatch(setComparisonTableCollapsed(sections[Section.COMPARISON_TABLE])) dispatch(setTemplatePlotsCollapsed(sections[Section.TEMPLATE_PLOTS])) @@ -59,9 +53,6 @@ export const feedStore = ( dispatch(initialize()) for (const key of Object.keys(data.data)) { switch (key) { - case PlotsDataKeys.CHECKPOINT: - dispatch(updateCheckpointPlots(data.data[key] as CheckpointPlotsData)) - continue case PlotsDataKeys.CUSTOM: dispatch(updateCustomPlots(data.data[key] as CustomPlotsData)) continue diff --git a/webview/src/plots/components/checkpointPlots/CheckpointPlot.tsx b/webview/src/plots/components/checkpointPlots/CheckpointPlot.tsx deleted file mode 100644 index f55c22c5b4..0000000000 --- a/webview/src/plots/components/checkpointPlots/CheckpointPlot.tsx +++ /dev/null @@ -1,59 +0,0 @@ -import { ColorScale, Section } from 'dvc/src/plots/webview/contract' -import React, { useMemo, useEffect, useState } from 'react' -import { useSelector } from 'react-redux' -import { createSpec } from './util' -import { changeDisabledDragIds, changeSize } from './checkpointPlotsSlice' -import { ZoomablePlot } from '../ZoomablePlot' -import styles from '../styles.module.scss' -import { withScale } from '../../../util/styles' -import { plotDataStore } from '../plotDataStore' -import { PlotsState } from '../../store' - -interface CheckpointPlotProps { - id: string - colors: ColorScale -} - -export const CheckpointPlot: React.FC = ({ - id, - colors -}) => { - const plotSnapshot = useSelector( - (state: PlotsState) => state.checkpoint.plotsSnapshots[id] - ) - const [plot, setPlot] = useState(plotDataStore[Section.CHECKPOINT_PLOTS][id]) - const nbItemsPerRow = useSelector( - (state: PlotsState) => state.checkpoint.nbItemsPerRow - ) - - const spec = useMemo(() => { - const title = plot?.title - if (!title) { - return {} - } - return createSpec(title, colors) - }, [plot?.title, colors]) - - useEffect(() => { - setPlot(plotDataStore[Section.CHECKPOINT_PLOTS][id]) - }, [plotSnapshot, id]) - - if (!plot) { - return null - } - - const key = `plot-${id}` - - return ( -
- -
- ) -} diff --git a/webview/src/plots/components/checkpointPlots/CheckpointPlots.tsx b/webview/src/plots/components/checkpointPlots/CheckpointPlots.tsx deleted file mode 100644 index 5b6048b64d..0000000000 --- a/webview/src/plots/components/checkpointPlots/CheckpointPlots.tsx +++ /dev/null @@ -1,117 +0,0 @@ -import React, { DragEvent, useEffect, useState } from 'react' -import { useSelector } from 'react-redux' -import cx from 'classnames' -import { ColorScale } from 'dvc/src/plots/webview/contract' -import { MessageFromWebviewType } from 'dvc/src/webview/contract' -import { performSimpleOrderedUpdate } from 'dvc/src/util/array' -import { CheckpointPlot } from './CheckpointPlot' -import styles from '../styles.module.scss' -import { EmptyState } from '../../../shared/components/emptyState/EmptyState' -import { - DragDropContainer, - WrapperProps -} from '../../../shared/components/dragDrop/DragDropContainer' -import { sendMessage } from '../../../shared/vscode' -import { DropTarget } from '../DropTarget' -import { VirtualizedGrid } from '../../../shared/components/virtualizedGrid/VirtualizedGrid' -import { shouldUseVirtualizedGrid } from '../util' -import { PlotsState } from '../../store' -import { changeOrderWithDraggedInfo } from '../../../util/array' -import { LoadingSection, sectionIsLoading } from '../LoadingSection' - -interface CheckpointPlotsProps { - plotsIds: string[] - colors: ColorScale -} - -export const CheckpointPlots: React.FC = ({ - plotsIds, - colors -}) => { - const [order, setOrder] = useState(plotsIds) - const { nbItemsPerRow, hasData, disabledDragPlotIds } = useSelector( - (state: PlotsState) => state.checkpoint - ) - const [onSection, setOnSection] = useState(false) - const draggedRef = useSelector( - (state: PlotsState) => state.dragAndDrop.draggedRef - ) - - const selectedRevisions = useSelector( - (state: PlotsState) => state.webview.selectedRevisions - ) - - useEffect(() => { - setOrder(pastOrder => performSimpleOrderedUpdate(pastOrder, plotsIds)) - }, [plotsIds]) - - const setMetricOrder = (order: string[]): void => { - setOrder(order) - sendMessage({ - payload: order, - type: MessageFromWebviewType.REORDER_PLOTS_METRICS - }) - } - - if (sectionIsLoading(selectedRevisions)) { - return - } - - if (!hasData) { - return No Plots to Display - } - - const items = order.map(plot => ( -
- -
- )) - - const useVirtualizedGrid = shouldUseVirtualizedGrid( - items.length, - nbItemsPerRow - ) - - const handleDropAtTheEnd = () => { - setMetricOrder(changeOrderWithDraggedInfo(order, draggedRef)) - } - - const handleDragOver = (e: DragEvent) => { - e.preventDefault() - setOnSection(true) - } - - return items.length > 0 ? ( -
setOnSection(true)} - onDragLeave={() => setOnSection(false)} - onDragOver={handleDragOver} - onDrop={handleDropAtTheEnd} - > - } - wrapperComponent={ - useVirtualizedGrid - ? { - component: VirtualizedGrid as React.FC, - props: { nbItemsPerRow } - } - : undefined - } - parentDraggedOver={onSection} - /> -
- ) : ( - No Metrics Selected - ) -} diff --git a/webview/src/plots/components/checkpointPlots/CheckpointPlotsWrapper.tsx b/webview/src/plots/components/checkpointPlots/CheckpointPlotsWrapper.tsx deleted file mode 100644 index 8c96b8c290..0000000000 --- a/webview/src/plots/components/checkpointPlots/CheckpointPlotsWrapper.tsx +++ /dev/null @@ -1,49 +0,0 @@ -import { Section } from 'dvc/src/plots/webview/contract' -import { MessageFromWebviewType } from 'dvc/src/webview/contract' -import React, { useEffect, useState } from 'react' -import { useSelector } from 'react-redux' -import { CheckpointPlots } from './CheckpointPlots' -import { PlotsContainer } from '../PlotsContainer' -import { sendMessage } from '../../../shared/vscode' -import { PlotsState } from '../../store' - -export const CheckpointPlotsWrapper: React.FC = () => { - const { plotsIds, nbItemsPerRow, selectedMetrics, isCollapsed, colors } = - useSelector((state: PlotsState) => state.checkpoint) - const [metrics, setMetrics] = useState([]) - const [selectedPlots, setSelectedPlots] = useState([]) - - useEffect(() => { - setMetrics([...plotsIds].sort()) - setSelectedPlots(selectedMetrics || []) - }, [plotsIds, selectedMetrics, setSelectedPlots, setMetrics]) - - const setSelectedMetrics = (metrics: string[]) => { - setSelectedPlots(metrics) - sendMessage({ - payload: metrics, - type: MessageFromWebviewType.TOGGLE_METRIC - }) - } - - const menu = - plotsIds.length > 0 - ? { - plots: metrics, - selectedPlots, - setSelectedPlots: setSelectedMetrics - } - : undefined - - return ( - - - - ) -} diff --git a/webview/src/plots/components/checkpointPlots/checkpointPlotsSlice.ts b/webview/src/plots/components/checkpointPlots/checkpointPlotsSlice.ts deleted file mode 100644 index f1e3e35adf..0000000000 --- a/webview/src/plots/components/checkpointPlots/checkpointPlotsSlice.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { createSlice, PayloadAction } from '@reduxjs/toolkit' -import { - CheckpointPlotsData, - DEFAULT_HEIGHT, - DEFAULT_SECTION_COLLAPSED, - DEFAULT_SECTION_NB_ITEMS_PER_ROW, - Section -} from 'dvc/src/plots/webview/contract' -import { addPlotsWithSnapshots, removePlots } from '../plotDataStore' - -export interface CheckpointPlotsState - extends Omit { - isCollapsed: boolean - hasData: boolean - plotsIds: string[] - plotsSnapshots: { [key: string]: string } - disabledDragPlotIds: string[] -} - -export const checkpointPlotsInitialState: CheckpointPlotsState = { - colors: { domain: [], range: [] }, - disabledDragPlotIds: [], - hasData: false, - height: DEFAULT_HEIGHT[Section.CHECKPOINT_PLOTS], - isCollapsed: DEFAULT_SECTION_COLLAPSED[Section.CHECKPOINT_PLOTS], - nbItemsPerRow: DEFAULT_SECTION_NB_ITEMS_PER_ROW[Section.CHECKPOINT_PLOTS], - plotsIds: [], - plotsSnapshots: {}, - selectedMetrics: [] -} - -export const checkpointPlotsSlice = createSlice({ - initialState: checkpointPlotsInitialState, - name: 'checkpoint', - reducers: { - changeDisabledDragIds: (state, action: PayloadAction) => { - state.disabledDragPlotIds = action.payload - }, - changeSize: (state, action: PayloadAction) => { - state.nbItemsPerRow = action.payload - }, - setCollapsed: (state, action: PayloadAction) => { - state.isCollapsed = action.payload - }, - update: (state, action: PayloadAction) => { - if (!action.payload) { - return checkpointPlotsInitialState - } - const { plots, ...statePayload } = action.payload - const plotsIds = plots?.map(plot => plot.id) || [] - const snapShots = addPlotsWithSnapshots(plots, Section.CHECKPOINT_PLOTS) - removePlots(plotsIds, Section.CHECKPOINT_PLOTS) - return { - ...state, - ...statePayload, - hasData: !!action.payload, - plotsIds: plots?.map(plot => plot.id) || [], - plotsSnapshots: snapShots - } - } - } -}) - -export const { update, setCollapsed, changeSize, changeDisabledDragIds } = - checkpointPlotsSlice.actions - -export default checkpointPlotsSlice.reducer diff --git a/webview/src/plots/components/checkpointPlots/util.ts b/webview/src/plots/components/checkpointPlots/util.ts deleted file mode 100644 index 94e099555b..0000000000 --- a/webview/src/plots/components/checkpointPlots/util.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { VisualizationSpec } from 'react-vega' -import { ColorScale } from 'dvc/src/plots/webview/contract' - -export const createSpec = ( - title: string, - scale?: ColorScale -): VisualizationSpec => - ({ - $schema: 'https://vega.github.io/schema/vega-lite/v5.json', - data: { name: 'values' }, - encoding: { - color: { - field: 'group', - legend: { disable: true }, - scale, - title: 'rev', - type: 'nominal' - }, - x: { - axis: { format: '0d', tickMinStep: 1 }, - field: 'iteration', - title: 'iteration', - type: 'quantitative' - }, - y: { - field: 'y', - scale: { zero: false }, - title, - type: 'quantitative' - } - }, - height: 'container', - layer: [ - { - layer: [ - { mark: { type: 'line' } }, - { - mark: { type: 'point' }, - transform: [ - { - filter: { empty: false, param: 'hover' } - } - ] - } - ] - }, - { - encoding: { - opacity: { value: 0 }, - tooltip: [ - { field: 'group', title: 'name' }, - { - field: 'y', - title: title.slice(Math.max(0, title.indexOf(':') + 1)), - type: 'quantitative' - } - ] - }, - mark: { type: 'rule' }, - params: [ - { - name: 'hover', - select: { - clear: 'mouseout', - fields: ['iteration', 'y'], - nearest: true, - on: 'mouseover', - type: 'point' - } - } - ] - }, - { - encoding: { - color: { field: 'group', scale }, - x: { aggregate: 'max', field: 'iteration', type: 'quantitative' }, - y: { - aggregate: { argmax: 'iteration' }, - field: 'y', - type: 'quantitative' - } - }, - mark: { stroke: null, type: 'circle' } - } - ], - transform: [ - { - as: 'y', - calculate: "format(datum['y'],'.5f')" - } - ], - width: 'container' - } as VisualizationSpec) diff --git a/webview/src/plots/hooks/useGetPlot.ts b/webview/src/plots/hooks/useGetPlot.ts index 955390f0cf..69c8bd2c16 100644 --- a/webview/src/plots/hooks/useGetPlot.ts +++ b/webview/src/plots/hooks/useGetPlot.ts @@ -10,16 +10,8 @@ import { PlainObject, VisualizationSpec } from 'react-vega' import { plotDataStore } from '../components/plotDataStore' import { PlotsState } from '../store' -const getStoreSection = (section: Section) => { - switch (section) { - case Section.CHECKPOINT_PLOTS: - return 'checkpoint' - case Section.TEMPLATE_PLOTS: - return 'template' - default: - return 'custom' - } -} +const getStoreSection = (section: Section) => + section === Section.TEMPLATE_PLOTS ? 'template' : 'custom' export const useGetPlot = ( section: Section, diff --git a/webview/src/plots/store.ts b/webview/src/plots/store.ts index 9686b1fda8..f5deecb327 100644 --- a/webview/src/plots/store.ts +++ b/webview/src/plots/store.ts @@ -1,5 +1,4 @@ import { configureStore } from '@reduxjs/toolkit' -import checkpointPlotsReducer from './components/checkpointPlots/checkpointPlotsSlice' import comparisonTableReducer from './components/comparisonTable/comparisonTableSlice' import templatePlotsReducer from './components/templatePlots/templatePlotsSlice' import customPlotsReducer from './components/customPlots/customPlotsSlice' @@ -8,7 +7,6 @@ import ribbonReducer from './components/ribbon/ribbonSlice' import dragAndDropReducer from '../shared/components/dragDrop/dragDropSlice' export const plotsReducers = { - checkpoint: checkpointPlotsReducer, comparison: comparisonTableReducer, custom: customPlotsReducer, dragAndDrop: dragAndDropReducer, diff --git a/webview/src/stories/Plots.stories.tsx b/webview/src/stories/Plots.stories.tsx index fbaa6c5cec..5ad6e6b298 100644 --- a/webview/src/stories/Plots.stories.tsx +++ b/webview/src/stories/Plots.stories.tsx @@ -28,18 +28,6 @@ import '../plots/components/styles.module.scss' import { feedStore } from '../plots/components/App' import { plotsReducers } from '../plots/store' -const smallCheckpointPlotsFixture = { - ...checkpointPlotsFixture, - nbItemsPerRow: PlotNumberOfItemsPerRow.THREE, - plots: checkpointPlotsFixture.plots.map(plot => ({ - ...plot, - title: truncateVerticalTitle( - plot.title, - PlotNumberOfItemsPerRow.THREE - ) as string - })) -} - const manyCheckpointPlots = ( length: number, size = PlotNumberOfItemsPerRow.TWO @@ -218,7 +206,6 @@ AllLarge.parameters = CHROMATIC_VIEWPORTS export const AllSmall = Template.bind({}) AllSmall.args = { data: { - checkpoint: smallCheckpointPlotsFixture, comparison: { ...comparisonPlotsFixture, nbItemsPerRow: PlotNumberOfItemsPerRow.THREE From c32500807f3bfee40c38b60d4dc755657334d3f5 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Tue, 7 Mar 2023 09:40:07 -0600 Subject: [PATCH 04/40] Fix jest tests --- extension/src/plots/model/collect.test.ts | 138 +++-- extension/src/plots/model/collect.ts | 41 +- extension/src/plots/model/index.test.ts | 34 +- extension/src/plots/model/index.ts | 61 -- extension/src/plots/model/quickPick.test.ts | 106 +++- extension/src/plots/model/quickPick.ts | 1 + extension/src/plots/webview/contract.ts | 6 - extension/src/plots/webview/messages.ts | 27 - .../fixtures/expShow/base/checkpointPlots.ts | 107 ---- .../test/fixtures/expShow/base/customPlots.ts | 42 ++ .../test/suite/experiments/model/tree.test.ts | 15 +- extension/src/test/suite/plots/index.test.ts | 48 +- extension/src/test/suite/plots/util.ts | 19 +- webview/src/plots/components/App.test.tsx | 570 ++++++------------ .../src/plots/components/PlotsContainer.tsx | 11 - webview/src/plots/components/plotDataStore.ts | 4 +- webview/src/plots/hooks/useGetPlot.ts | 3 +- webview/src/stories/Plots.stories.tsx | 46 +- 18 files changed, 435 insertions(+), 844 deletions(-) delete mode 100644 extension/src/test/fixtures/expShow/base/checkpointPlots.ts diff --git a/extension/src/plots/model/collect.test.ts b/extension/src/plots/model/collect.test.ts index 72425d3d2e..679402b3e3 100644 --- a/extension/src/plots/model/collect.test.ts +++ b/extension/src/plots/model/collect.test.ts @@ -1,25 +1,25 @@ import { join } from 'path' -import omit from 'lodash.omit' import isEmpty from 'lodash.isempty' import { collectData, - collectCheckpointPlotsData, collectTemplates, collectOverrideRevisionDetails, collectCustomPlotsData } from './collect' import plotsDiffFixture from '../../test/fixtures/plotsDiff/output' -import expShowFixture from '../../test/fixtures/expShow/base/output' -import modifiedFixture from '../../test/fixtures/expShow/modified/output' -import checkpointPlotsFixture from '../../test/fixtures/expShow/base/checkpointPlots' import customPlotsFixture from '../../test/fixtures/expShow/base/customPlots' import { - ExperimentsOutput, ExperimentStatus, EXPERIMENT_WORKSPACE_ID } from '../../cli/dvc/contract' -import { definedAndNonEmpty, sameContents } from '../../util/array' -import { CustomPlotType, TemplatePlot } from '../webview/contract' +import { sameContents } from '../../util/array' +import { + CustomPlot, + CustomPlotData, + CustomPlotType, + MetricVsParamPlotData, + TemplatePlot +} from '../webview/contract' import { getCLICommitId } from '../../test/fixtures/plotsDiff/util' import { SelectedExperimentWithColor } from '../../experiments/model' import { Experiment } from '../../experiments/webview/contract' @@ -29,7 +29,24 @@ const logsLossPath = join('logs', 'loss.tsv') const logsLossPlot = (plotsDiffFixture[logsLossPath][0] || {}) as TemplatePlot describe('collectCustomPlotsData', () => { - it('should return the expected data from the text fixture', () => { + it('should return the expected data from the test fixture', () => { + const expectedOutput: CustomPlot[] = customPlotsFixture.plots.map( + ({ type, metric, id, values, ...plot }: CustomPlotData) => + type === CustomPlotType.CHECKPOINT + ? { + id, + metric, + type, + values + } + : { + id, + metric, + param: (plot as MetricVsParamPlotData).param, + type, + values + } + ) const data = collectCustomPlotsData( [ { @@ -41,9 +58,56 @@ describe('collectCustomPlotsData', () => { metric: 'metrics:summary.json:accuracy', param: 'params:params.yaml:epochs', type: CustomPlotType.METRIC_VS_PARAM + }, + { + metric: 'metrics:summary.json:loss', + type: CustomPlotType.CHECKPOINT + }, + { + metric: 'metrics:summary.json:accuracy', + type: CustomPlotType.CHECKPOINT } ], - {}, + { + 'summary.json:accuracy': { + id: 'custom-summary.json:accuracy', + metric: 'summary.json:accuracy', + type: CustomPlotType.CHECKPOINT, + values: [ + { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, + { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, + { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, + { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, + { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, + { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, + { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, + { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, + { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, + { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, + { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, + { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } + ] + }, + 'summary.json:loss': { + id: 'custom-summary.json:loss', + metric: 'summary.json:loss', + type: CustomPlotType.CHECKPOINT, + values: [ + { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, + { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, + { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, + { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, + { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, + { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, + { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, + { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, + { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, + { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, + { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, + { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } + ] + } + }, [ { id: '12345', @@ -83,59 +147,7 @@ describe('collectCustomPlotsData', () => { } ] ) - expect(data).toStrictEqual(customPlotsFixture.plots) - }) -}) - -describe('collectCheckpointPlotsData', () => { - it('should return the expected data from the test fixture', () => { - const data = collectCheckpointPlotsData(expShowFixture) - expect(data).toStrictEqual( - checkpointPlotsFixture.plots.map(({ id, values }) => ({ id, values })) - ) - }) - - it('should provide a continuous series for a modified experiment', () => { - const data = collectCheckpointPlotsData(modifiedFixture) - - expect(definedAndNonEmpty(data)).toBeTruthy() - - for (const { values } of data || []) { - const initialExperiment = values.filter( - point => point.group === 'exp-908bd' - ) - const modifiedExperiment = values.find( - point => point.group === 'exp-01b3a' - ) - - const lastIterationInitial = initialExperiment?.slice(-1)[0] - const firstIterationModified = modifiedExperiment - - expect(lastIterationInitial).not.toStrictEqual(firstIterationModified) - expect(omit(lastIterationInitial, 'group')).toStrictEqual( - omit(firstIterationModified, 'group') - ) - - const baseExperiment = values.filter(point => point.group === 'exp-920fc') - const restartedExperiment = values.find( - point => point.group === 'exp-9bc1b' - ) - - const iterationRestartedFrom = baseExperiment?.slice(5)[0] - const firstIterationAfterRestart = restartedExperiment - - expect(iterationRestartedFrom).not.toStrictEqual( - firstIterationAfterRestart - ) - expect(omit(iterationRestartedFrom, 'group')).toStrictEqual( - omit(firstIterationAfterRestart, 'group') - ) - } - }) - - it('should return undefined given no input', () => { - const data = collectCheckpointPlotsData({} as ExperimentsOutput) - expect(data).toBeUndefined() + expect(data).toStrictEqual(expectedOutput) }) }) diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 0571521e1b..1fb3e4027c 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -218,47 +218,12 @@ const collectFromExperimentsObject = ( } } -export const collectCheckpointPlotsData = ( - data: ExperimentsOutput -): CheckpointPlot[] | undefined => { - const acc = { - iterations: {}, - plots: new Map() - } - - for (const { baseline, ...experimentsObject } of Object.values( - omit(data, EXPERIMENT_WORKSPACE_ID) - )) { - const commit = transformExperimentData(baseline) - - if (commit) { - collectFromExperimentsObject(acc, experimentsObject) - } - } - - if (acc.plots.size === 0) { - return - } - - const plotsData: CheckpointPlot[] = [] - - for (const [key, value] of acc.plots.entries()) { - plotsData.push({ - id: decodeColumn(key), - metric: decodeColumn(key), - type: CustomPlotType.CHECKPOINT, - values: value - }) - } - - return plotsData -} - export const getCustomPlotId = (plot: CustomPlotsOrderValue) => plot.type === CustomPlotType.CHECKPOINT ? `custom-${plot.metric}` : `custom-${plot.metric}-${plot.param}` +// TBD untested... export const collectCustomCheckpointPlotData = ( data: ExperimentsOutput ): { [metric: string]: CheckpointPlot } => { @@ -340,11 +305,11 @@ const collectMetricVsParamPlotData = ( // TBD it will probably be easier and/or faster to get the data from // experiments vs the output... export const collectCustomPlotsData = ( - plotsOrderValue: CustomPlotsOrderValue[], + plotsOrderValues: CustomPlotsOrderValue[], checkpointPlots: { [metric: string]: CheckpointPlot }, experiments: Experiment[] ): CustomPlot[] => { - return plotsOrderValue + return plotsOrderValues .map((plotOrderValue): CustomPlot => { if (isCustomPlotOrderCheckpointValue(plotOrderValue)) { const { metric } = plotOrderValue diff --git a/extension/src/plots/model/index.test.ts b/extension/src/plots/model/index.test.ts index 20671e03cc..4b74e4dbdc 100644 --- a/extension/src/plots/model/index.test.ts +++ b/extension/src/plots/model/index.test.ts @@ -54,30 +54,14 @@ describe('plotsModel', () => { expect(model.getSelectedMetrics()).toStrictEqual(newSelectedMetrics) }) - it('should update the persisted selected metrics when calling setSelectedMetrics', () => { - const mementoUpdateSpy = jest.spyOn(memento, 'update') - const newSelectedMetrics = ['one', 'two', 'four', 'hundred'] - - model.setSelectedMetrics(newSelectedMetrics) - - expect(mementoUpdateSpy).toHaveBeenCalledTimes(2) - expect(mementoUpdateSpy).toHaveBeenCalledWith( - PersistenceKey.PLOT_SELECTED_METRICS + exampleDvcRoot, - newSelectedMetrics - ) - }) - it('should change the plotSize when calling setPlotSize', () => { - expect(model.getNbItemsPerRow(Section.CHECKPOINT_PLOTS)).toStrictEqual( + expect(model.getNbItemsPerRow(Section.CUSTOM_PLOTS)).toStrictEqual( PlotNumberOfItemsPerRow.TWO ) - model.setNbItemsPerRow( - Section.CHECKPOINT_PLOTS, - PlotNumberOfItemsPerRow.ONE - ) + model.setNbItemsPerRow(Section.CUSTOM_PLOTS, PlotNumberOfItemsPerRow.ONE) - expect(model.getNbItemsPerRow(Section.CHECKPOINT_PLOTS)).toStrictEqual( + expect(model.getNbItemsPerRow(Section.CUSTOM_PLOTS)).toStrictEqual( PlotNumberOfItemsPerRow.ONE ) }) @@ -85,17 +69,14 @@ describe('plotsModel', () => { it('should update the persisted plot size when calling setPlotSize', () => { const mementoUpdateSpy = jest.spyOn(memento, 'update') - model.setNbItemsPerRow( - Section.CHECKPOINT_PLOTS, - PlotNumberOfItemsPerRow.TWO - ) + model.setNbItemsPerRow(Section.CUSTOM_PLOTS, PlotNumberOfItemsPerRow.TWO) expect(mementoUpdateSpy).toHaveBeenCalledTimes(1) expect(mementoUpdateSpy).toHaveBeenCalledWith( PersistenceKey.PLOT_NB_ITEMS_PER_ROW + exampleDvcRoot, { ...DEFAULT_SECTION_NB_ITEMS_PER_ROW, - [Section.CHECKPOINT_PLOTS]: PlotNumberOfItemsPerRow.TWO + [Section.CUSTOM_PLOTS]: PlotNumberOfItemsPerRow.TWO } ) }) @@ -105,12 +86,11 @@ describe('plotsModel', () => { expect(model.getSectionCollapsed()).toStrictEqual(DEFAULT_SECTION_COLLAPSED) - model.setSectionCollapsed({ [Section.CHECKPOINT_PLOTS]: true }) + model.setSectionCollapsed({ [Section.CUSTOM_PLOTS]: true }) const expectedSectionCollapsed = { - [Section.CHECKPOINT_PLOTS]: true, [Section.TEMPLATE_PLOTS]: false, - [Section.CUSTOM_PLOTS]: false, + [Section.CUSTOM_PLOTS]: true, [Section.COMPARISON_TABLE]: false } diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 506c2f4fbd..a87252c2ce 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -2,7 +2,6 @@ import { Memento } from 'vscode' import isEmpty from 'lodash.isempty' import isEqual from 'lodash.isequal' import { - collectCheckpointPlotsData, collectData, collectSelectedTemplatePlots, collectTemplates, @@ -20,7 +19,6 @@ import { getRevisionFirstThreeColumns } from './util' import { CustomPlotsOrderValue } from './custom' import { CheckpointPlot, - CheckpointPlotData, ComparisonPlots, Revision, ComparisonRevisionData, @@ -110,14 +108,6 @@ export class PlotsModel extends ModelWithPersistence { } public transformAndSetExperiments(data: ExperimentsOutput) { - const checkpointPlots = collectCheckpointPlotsData(data) - - if (!this.selectedMetrics && checkpointPlots) { - this.selectedMetrics = checkpointPlots.map(({ id }) => id) - } - - this.checkpointPlots = checkpointPlots - this.recreateCustomPlots(data) return this.removeStaleData() @@ -165,32 +155,6 @@ export class PlotsModel extends ModelWithPersistence { this.deferred.resolve() } - public getCheckpointPlots() { - if (!this.checkpointPlots) { - return - } - - const colors = getColorScale( - this.experiments - .getSelectedExperiments() - .map(({ displayColor, id: revision }) => ({ displayColor, revision })) - ) - - if (!colors) { - return - } - - const { domain: selectedExperiments } = colors - - return { - colors, - height: this.getHeight(Section.CHECKPOINT_PLOTS), - nbItemsPerRow: this.getNbItemsPerRow(Section.CHECKPOINT_PLOTS), - plots: this.getPlots(this.checkpointPlots, selectedExperiments), - selectedMetrics: this.getSelectedMetrics() - } - } - public getCustomPlots(): CustomPlotsData | undefined { if (!this.customPlots) { return @@ -512,31 +476,6 @@ export class PlotsModel extends ModelWithPersistence { return this.commitRevisions[label] || label } - private getPlots( - checkpointPlots: CheckpointPlot[], - selectedExperiments: string[] - ) { - return reorderObjectList( - this.metricOrder, - checkpointPlots.map(plot => { - const { id, values, type } = plot - return { - id, - metric: id, - type, - values: values.filter(value => - selectedExperiments.includes(value.group) - ), - yTitle: truncateVerticalTitle( - id, - this.getNbItemsPerRow(Section.CHECKPOINT_PLOTS) - ) as string - } - }), - 'id' - ) - } - private getCustomPlotData( plots: CustomPlot[], selectedExperiments: string[] | undefined diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index 3e195e9117..d118313ec7 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -1,5 +1,10 @@ import { CustomPlotsOrderValue } from './custom' -import { pickCustomPlots, pickMetricAndParam } from './quickPick' +import { + pickCustomPlots, + pickCustomPlotType, + pickMetric, + pickMetricAndParam +} from './quickPick' import { quickPickManyValues, quickPickValue } from '../../vscode/quickPick' import { Title } from '../../vscode/title' import { Toast } from '../../vscode/toast' @@ -30,14 +35,13 @@ describe('pickCustomPlots', () => { it('should return the selected plots', async () => { const selectedPlots = [ - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-metrics:summary.json:loss', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' ] - const mockedExperiments = [ + const mockedPlots = [ { metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout', - type: CustomPlotType.METRIC_VS_PARAM + type: CustomPlotType.CHECKPOINT }, { metric: 'metrics:summary.json:accuracy', @@ -53,7 +57,7 @@ describe('pickCustomPlots', () => { mockedQuickPickManyValues.mockResolvedValueOnce(selectedPlots) const picked = await pickCustomPlots( - mockedExperiments, + mockedPlots, 'There are no plots to remove.', { title: Title.SELECT_CUSTOM_PLOTS_TO_REMOVE } ) @@ -63,20 +67,21 @@ describe('pickCustomPlots', () => { expect(mockedQuickPickManyValues).toHaveBeenCalledWith( [ { - description: - 'metrics:summary.json:loss vs params:params.yaml:dropout', - label: 'loss vs dropout', - value: 'custom-metrics:summary.json:loss-params:params.yaml:dropout' + description: 'Trend Plot', + detail: 'metrics:summary.json:loss', + label: 'loss', + value: 'custom-metrics:summary.json:loss' }, { - description: - 'metrics:summary.json:accuracy vs params:params.yaml:epochs', + description: 'Metric Vs Param Plot', + detail: 'metrics:summary.json:accuracy vs params:params.yaml:epochs', label: 'accuracy vs epochs', value: 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' }, { - description: + description: 'Metric Vs Param Plot', + detail: 'metrics:summary.json:learning_rate vs param:summary.json:process.threshold', label: 'learning_rate vs threshold', value: @@ -88,6 +93,37 @@ describe('pickCustomPlots', () => { }) }) +describe('pickCustomPlotType', () => { + it('should return a chosen custom plot type', async () => { + const expectedType = CustomPlotType.CHECKPOINT + mockedQuickPickValue.mockResolvedValueOnce(expectedType) + + const picked = await pickCustomPlotType() + + expect(picked).toStrictEqual(expectedType) + expect(mockedQuickPickValue).toHaveBeenCalledTimes(1) + expect(mockedQuickPickValue).toHaveBeenCalledWith( + [ + { + description: + 'A linear plot that compares a chosen metric and param with current experiments.', + label: 'Metric Vs Param', + value: CustomPlotType.METRIC_VS_PARAM + }, + { + description: + 'A linear plot that shows how a chosen metric changes over selected experiments.', + label: 'Trend', + value: CustomPlotType.CHECKPOINT + } + ], + { + title: Title.SELECT_PLOT_TYPE_CUSTOM_PLOT + } + ) + }) +}) + describe('pickMetricAndParam', () => { it('should end early if there are no metrics or params available', async () => { mockedQuickPickValue.mockResolvedValueOnce(undefined) @@ -130,3 +166,47 @@ describe('pickMetricAndParam', () => { }) }) }) + +describe('pickMetric', () => { + it('should end early if there are no metrics or params available', async () => { + mockedQuickPickValue.mockResolvedValueOnce(undefined) + const undef = await pickMetric([]) + expect(undef).toBeUndefined() + expect(mockedShowError).toHaveBeenCalledTimes(1) + }) + + it('should return a metric', async () => { + const expectedMetric = { + label: 'loss', + path: 'metrics:summary.json:loss' + } + mockedQuickPickValue.mockResolvedValueOnce(expectedMetric) + const metric = await pickMetric([ + { ...expectedMetric, hasChildren: false, type: ColumnType.METRICS }, + { + hasChildren: false, + label: 'accuracy', + path: 'summary.json:accuracy', + type: ColumnType.METRICS + } + ]) + + expect(metric).toStrictEqual(expectedMetric.path) + expect(mockedQuickPickValue).toHaveBeenCalledTimes(1) + expect(mockedQuickPickValue).toHaveBeenCalledWith( + [ + { + description: 'metrics:summary.json:loss', + label: 'loss', + value: { label: 'loss', path: 'metrics:summary.json:loss' } + }, + { + description: 'summary.json:accuracy', + label: 'accuracy', + value: { label: 'accuracy', path: 'summary.json:accuracy' } + } + ], + { title: Title.SELECT_METRIC_CUSTOM_PLOT } + ) + }) +}) diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index 5b4f3583d8..57ccfd0cbb 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -29,6 +29,7 @@ const getMetricVsParamPlotItem = (metric: string, param: string) => { }) } } + const getCheckpointPlotItem = (metric: string) => { const splitMetric = splitColumnPath(metric) return { diff --git a/extension/src/plots/webview/contract.ts b/extension/src/plots/webview/contract.ts index ebb1decccb..d0e4061319 100644 --- a/extension/src/plots/webview/contract.ts +++ b/extension/src/plots/webview/contract.ts @@ -12,14 +12,12 @@ export const PlotNumberOfItemsPerRow = { /* eslint-enable sort-keys-fix/sort-keys-fix */ export enum Section { - CHECKPOINT_PLOTS = 'checkpoint-plots', TEMPLATE_PLOTS = 'template-plots', COMPARISON_TABLE = 'comparison-table', CUSTOM_PLOTS = 'custom-plots' } export const DEFAULT_SECTION_NB_ITEMS_PER_ROW = { - [Section.CHECKPOINT_PLOTS]: PlotNumberOfItemsPerRow.TWO, [Section.TEMPLATE_PLOTS]: PlotNumberOfItemsPerRow.TWO, [Section.COMPARISON_TABLE]: PlotNumberOfItemsPerRow.TWO, [Section.CUSTOM_PLOTS]: PlotNumberOfItemsPerRow.TWO @@ -27,14 +25,12 @@ export const DEFAULT_SECTION_NB_ITEMS_PER_ROW = { // Height is undefined by default because it is calculated by ratio of the width it'll fill (calculated by the webview) export const DEFAULT_HEIGHT = { - [Section.CHECKPOINT_PLOTS]: undefined, [Section.TEMPLATE_PLOTS]: undefined, [Section.COMPARISON_TABLE]: undefined, [Section.CUSTOM_PLOTS]: undefined } export const DEFAULT_SECTION_COLLAPSED = { - [Section.CHECKPOINT_PLOTS]: false, [Section.TEMPLATE_PLOTS]: false, [Section.COMPARISON_TABLE]: false, [Section.CUSTOM_PLOTS]: false @@ -183,7 +179,6 @@ export type ComparisonPlot = { export enum PlotsDataKeys { COMPARISON = 'comparison', - CHECKPOINT = 'checkpoint', CUSTOM = 'custom', HAS_UNSELECTED_PLOTS = 'hasUnselectedPlots', HAS_PLOTS = 'hasPlots', @@ -195,7 +190,6 @@ export enum PlotsDataKeys { export type PlotsData = | { [PlotsDataKeys.COMPARISON]?: PlotsComparisonData | null - [PlotsDataKeys.CHECKPOINT]?: CheckpointPlotsData | null [PlotsDataKeys.CUSTOM]?: CustomPlotsData | null [PlotsDataKeys.HAS_PLOTS]?: boolean [PlotsDataKeys.HAS_UNSELECTED_PLOTS]?: boolean diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index b7264fd60a..7ca5737e70 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -68,7 +68,6 @@ export class WebviewMessages { this.plots.getOverrideRevisionDetails() void this.getWebview()?.show({ - checkpoint: this.getCheckpointPlots(), comparison: this.getComparisonPlots(overrideComparison), custom: this.getCustomPlots(), hasPlots: !!this.paths.hasPaths(), @@ -79,18 +78,10 @@ export class WebviewMessages { }) } - public sendCheckpointPlotsMessage() { - void this.getWebview()?.show({ - checkpoint: this.getCheckpointPlots() - }) - } - public handleMessageFromWebview(message: MessageFromWebview) { switch (message.type) { case MessageFromWebviewType.ADD_CUSTOM_PLOT: return this.addCustomPlot() - case MessageFromWebviewType.TOGGLE_METRIC: - return this.setSelectedMetrics(message.payload) case MessageFromWebviewType.RESIZE_PLOTS: return this.setPlotSize( message.payload.section, @@ -124,11 +115,6 @@ export class WebviewMessages { } } - private setSelectedMetrics(metrics: string[]) { - this.plots.setSelectedMetrics(metrics) - this.sendCheckpointPlotsAndEvent(EventName.VIEWS_PLOTS_METRICS_SELECTED) - } - private setPlotSize( section: Section, nbItemsPerRow: number, @@ -368,15 +354,6 @@ export class WebviewMessages { ) } - private sendCheckpointPlotsAndEvent( - event: - | typeof EventName.VIEWS_REORDER_PLOTS_METRICS - | typeof EventName.VIEWS_PLOTS_METRICS_SELECTED - ) { - this.sendCheckpointPlotsMessage() - sendTelemetryEvent(event, undefined, undefined) - } - private sendSectionCollapsed() { void this.getWebview()?.show({ sectionCollapsed: this.plots.getSectionCollapsed() @@ -461,10 +438,6 @@ export class WebviewMessages { } } - private getCheckpointPlots() { - return this.plots.getCheckpointPlots() || null - } - private getCustomPlots() { return this.plots.getCustomPlots() || null } diff --git a/extension/src/test/fixtures/expShow/base/checkpointPlots.ts b/extension/src/test/fixtures/expShow/base/checkpointPlots.ts deleted file mode 100644 index c0d146e558..0000000000 --- a/extension/src/test/fixtures/expShow/base/checkpointPlots.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { copyOriginalColors } from '../../../../experiments/model/status/colors' -import { - CheckpointPlotsData, - CustomPlotType, - PlotNumberOfItemsPerRow -} from '../../../../plots/webview/contract' - -const colors = copyOriginalColors() - -const data: CheckpointPlotsData = { - colors: { - domain: ['exp-e7a67', 'test-branch', 'exp-83425'], - range: [colors[2], colors[3], colors[4]] - }, - plots: [ - { - id: 'custom-summary.json:loss', - metric: 'summary.json:loss', - values: [ - { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, - { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, - { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, - { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, - { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, - { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, - { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, - { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, - { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, - { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, - { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, - { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } - ], - type: CustomPlotType.CHECKPOINT, - yTitle: 'summary.json:loss' - }, - { - id: 'custom-summary.json:accuracy', - metric: 'summary.json:accuracy', - values: [ - { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, - { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, - { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, - { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, - { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, - { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, - { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, - { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, - { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, - { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, - { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, - { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } - ], - type: CustomPlotType.CHECKPOINT, - yTitle: 'summary.json:accuracy' - }, - { - id: 'custom-summary.json:val_loss', - metric: 'summary.json:val_loss', - values: [ - { group: 'exp-83425', iteration: 1, y: 1.9391471147537231 }, - { group: 'exp-83425', iteration: 2, y: 1.8825950622558594 }, - { group: 'exp-83425', iteration: 3, y: 1.827923059463501 }, - { group: 'exp-83425', iteration: 4, y: 1.7749212980270386 }, - { group: 'exp-83425', iteration: 5, y: 1.7233840227127075 }, - { group: 'exp-83425', iteration: 6, y: 1.7233840227127075 }, - { group: 'test-branch', iteration: 1, y: 1.9363881349563599 }, - { group: 'test-branch', iteration: 2, y: 1.8770883083343506 }, - { group: 'test-branch', iteration: 3, y: 1.8770883083343506 }, - { group: 'exp-e7a67', iteration: 1, y: 1.9979370832443237 }, - { group: 'exp-e7a67', iteration: 2, y: 1.9979370832443237 }, - { group: 'exp-e7a67', iteration: 3, y: 1.9979370832443237 } - ], - type: CustomPlotType.CHECKPOINT, - yTitle: 'summary.json:val_loss' - }, - { - id: 'custom-summary.json:val_accuracy', - metric: 'summary.json:val_accuracy', - values: [ - { group: 'exp-83425', iteration: 1, y: 0.49399998784065247 }, - { group: 'exp-83425', iteration: 2, y: 0.5550000071525574 }, - { group: 'exp-83425', iteration: 3, y: 0.6035000085830688 }, - { group: 'exp-83425', iteration: 4, y: 0.6414999961853027 }, - { group: 'exp-83425', iteration: 5, y: 0.6704000234603882 }, - { group: 'exp-83425', iteration: 6, y: 0.6704000234603882 }, - { group: 'test-branch', iteration: 1, y: 0.4970000088214874 }, - { group: 'test-branch', iteration: 2, y: 0.5608000159263611 }, - { group: 'test-branch', iteration: 3, y: 0.5608000159263611 }, - { group: 'exp-e7a67', iteration: 1, y: 0.4277999997138977 }, - { group: 'exp-e7a67', iteration: 2, y: 0.4277999997138977 }, - { group: 'exp-e7a67', iteration: 3, y: 0.4277999997138977 } - ], - type: CustomPlotType.CHECKPOINT, - yTitle: 'summary.json:val_accuracy' - } - ], - selectedMetrics: [ - 'summary.json:loss', - 'summary.json:accuracy', - 'summary.json:val_loss', - 'summary.json:val_accuracy' - ], - nbItemsPerRow: PlotNumberOfItemsPerRow.TWO, - height: undefined -} - -export default data diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index 3f1c5e384f..8f13a160a7 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -15,6 +15,8 @@ const data: CustomPlotsData = { plots: [ { id: 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + // TBD I don't think we actually need metric/param here + // since I think only title is used in in the front end metric: 'summary.json:loss', param: 'params.yaml:dropout', type: CustomPlotType.METRIC_VS_PARAM, @@ -60,6 +62,46 @@ const data: CustomPlotsData = { } ], yTitle: 'summary.json:accuracy' + }, + { + id: 'custom-summary.json:loss', + metric: 'summary.json:loss', + values: [ + { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, + { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, + { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, + { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, + { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, + { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, + { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, + { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, + { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, + { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, + { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, + { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } + ], + type: CustomPlotType.CHECKPOINT, + yTitle: 'summary.json:loss' + }, + { + id: 'custom-summary.json:accuracy', + metric: 'summary.json:accuracy', + values: [ + { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, + { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, + { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, + { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, + { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, + { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, + { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, + { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, + { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, + { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, + { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, + { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } + ], + type: CustomPlotType.CHECKPOINT, + yTitle: 'summary.json:accuracy' } ], nbItemsPerRow: PlotNumberOfItemsPerRow.TWO, diff --git a/extension/src/test/suite/experiments/model/tree.test.ts b/extension/src/test/suite/experiments/model/tree.test.ts index a88c27d0c9..c4ede12f00 100644 --- a/extension/src/test/suite/experiments/model/tree.test.ts +++ b/extension/src/test/suite/experiments/model/tree.test.ts @@ -23,8 +23,8 @@ import { RegisteredCliCommands, RegisteredCommands } from '../../../../commands/external' -import { buildPlots, getExpectedCheckpointPlotsData } from '../../plots/util' -import checkpointPlotsFixture from '../../../fixtures/expShow/base/checkpointPlots' +import { buildPlots, getExpectedCustomPlotsData } from '../../plots/util' +import customPlotsFixture from '../../../fixtures/expShow/base/customPlots' import expShowFixture from '../../../fixtures/expShow/base/output' import { ExperimentsTree } from '../../../../experiments/model/tree' import { @@ -45,6 +45,7 @@ import { WorkspaceExperiments } from '../../../../experiments/workspace' import { ExperimentItem } from '../../../../experiments/model/collect' import { EXPERIMENT_WORKSPACE_ID } from '../../../../cli/dvc/contract' import { DvcReader } from '../../../../cli/dvc/reader' +import { ColorScale } from '../../../../plots/webview/contract' suite('Experiments Tree Test Suite', () => { const disposable = getTimeSafeDisposer() @@ -59,8 +60,8 @@ suite('Experiments Tree Test Suite', () => { // eslint-disable-next-line sonarjs/cognitive-complexity describe('ExperimentsTree', () => { - const { colors } = checkpointPlotsFixture - const { domain, range } = colors + const { colors } = customPlotsFixture + const { domain, range } = colors as ColorScale it('should appear in the UI', async () => { await expect( @@ -78,7 +79,7 @@ suite('Experiments Tree Test Suite', () => { await webview.isReady() while (expectedDomain.length > 0) { - const expectedData = getExpectedCheckpointPlotsData( + const expectedData = getExpectedCustomPlotsData( expectedDomain, expectedRange ) @@ -127,7 +128,7 @@ suite('Experiments Tree Test Suite', () => { expect(selected, 'the experiment is now selected').to.equal(range[0]) expect(messageSpy, 'we no longer send null').to.be.calledWithMatch( - getExpectedCheckpointPlotsData(expectedDomain, expectedRange) + getExpectedCustomPlotsData(expectedDomain, expectedRange) ) }).timeout(WEBVIEW_TEST_TIMEOUT) @@ -263,7 +264,7 @@ suite('Experiments Tree Test Suite', () => { messageSpy, 'a message is sent with colors for the currently selected experiments' ).to.be.calledWithMatch( - getExpectedCheckpointPlotsData([selectedDisplayName], [selectedColor]) + getExpectedCustomPlotsData([selectedDisplayName], [selectedColor]) ) }).timeout(WEBVIEW_TEST_TIMEOUT) diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index b7c4110a7b..b6725b2fef 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -7,7 +7,6 @@ import { restore, spy, stub } from 'sinon' import { buildPlots } from '../plots/util' import { Disposable } from '../../../extension' import expShowFixtureWithoutErrors from '../../fixtures/expShow/base/noErrors' -import checkpointPlotsFixture from '../../fixtures/expShow/base/checkpointPlots' import customPlotsFixture from '../../fixtures/expShow/base/customPlots' import plotsDiffFixture from '../../fixtures/plotsDiff/output' import multiSourcePlotsDiffFixture from '../../fixtures/plotsDiff/multiSource' @@ -192,48 +191,6 @@ suite('Plots Test Suite', () => { ) }) - it('should handle a set selected metrics message from the webview', async () => { - const { plots, plotsModel, messageSpy } = await buildPlots( - disposable, - plotsDiffFixture - ) - - const webview = await plots.showWebview() - - const mockSendTelemetryEvent = stub(Telemetry, 'sendTelemetryEvent') - const mockMessageReceived = getMessageReceivedEmitter(webview) - - const mockSetSelectedMetrics = spy(plotsModel, 'setSelectedMetrics') - const mockSelectedMetrics = ['summary.json:loss'] - - messageSpy.resetHistory() - mockMessageReceived.fire({ - payload: mockSelectedMetrics, - type: MessageFromWebviewType.TOGGLE_METRIC - }) - - expect(mockSetSelectedMetrics).to.be.calledOnce - expect(mockSetSelectedMetrics).to.be.calledWithExactly( - mockSelectedMetrics - ) - expect(messageSpy).to.be.calledOnce - expect( - messageSpy, - "should update the webview's checkpoint plot state" - ).to.be.calledWithExactly({ - checkpoint: { - ...checkpointPlotsFixture, - selectedMetrics: mockSelectedMetrics - } - }) - expect(mockSendTelemetryEvent).to.be.calledOnce - expect(mockSendTelemetryEvent).to.be.calledWithExactly( - EventName.VIEWS_PLOTS_METRICS_SELECTED, - undefined, - undefined - ) - }).timeout(WEBVIEW_TEST_TIMEOUT) - it('should handle a section resized message from the webview', async () => { const { plots, plotsModel } = await buildPlots(disposable) @@ -281,7 +238,7 @@ suite('Plots Test Suite', () => { const mockMessageReceived = getMessageReceivedEmitter(webview) const mockSetSectionCollapsed = spy(plotsModel, 'setSectionCollapsed') - const mockSectionCollapsed = { [Section.CHECKPOINT_PLOTS]: true } + const mockSectionCollapsed = { [Section.CUSTOM_PLOTS]: true } messageSpy.resetHistory() mockMessageReceived.fire({ @@ -667,19 +624,16 @@ suite('Plots Test Suite', () => { expect(mockPlotsDiff).to.be.called const { - checkpoint: checkpointData, comparison: comparisonData, sectionCollapsed, template: templateData } = getFirstArgOfLastCall(messageSpy) - expect(checkpointData).to.deep.equal(checkpointPlotsFixture) expect(comparisonData).to.deep.equal(comparisonPlotsFixture) expect(sectionCollapsed).to.deep.equal(DEFAULT_SECTION_COLLAPSED) expect(templateData).to.deep.equal(templatePlotsFixture) const expectedPlotsData: TPlotsData = { - checkpoint: checkpointPlotsFixture, comparison: comparisonPlotsFixture, custom: customPlotsFixture, hasPlots: true, diff --git a/extension/src/test/suite/plots/util.ts b/extension/src/test/suite/plots/util.ts index 00ebb17230..980ebe983f 100644 --- a/extension/src/test/suite/plots/util.ts +++ b/extension/src/test/suite/plots/util.ts @@ -2,7 +2,7 @@ import { Disposer } from '@hediet/std/disposable' import { stub } from 'sinon' import * as FileSystem from '../../../fileSystem' import expShowFixtureWithoutErrors from '../../fixtures/expShow/base/noErrors' -import checkpointPlotsFixture from '../../fixtures/expShow/base/checkpointPlots' +import customPlotsFixture from '../../fixtures/expShow/base/customPlots' import { Plots } from '../../../plots' import { buildMockMemento, dvcDemoPath } from '../../util' import { WorkspacePlots } from '../../../plots/workspace' @@ -22,6 +22,7 @@ import { WebviewMessages } from '../../../plots/webview/messages' import { ExperimentsModel } from '../../../experiments/model' import { Experiment } from '../../../experiments/webview/contract' import { EXPERIMENT_WORKSPACE_ID } from '../../../cli/dvc/contract' +import { CustomPlotType } from '../../../plots/webview/contract' export const buildPlots = async ( disposer: Disposer, @@ -127,12 +128,11 @@ export const buildWorkspacePlots = (disposer: Disposer) => { } } -export const getExpectedCheckpointPlotsData = ( +export const getExpectedCustomPlotsData = ( domain: string[], range: Color[] ) => { - const { plots, selectedMetrics, nbItemsPerRow, height } = - checkpointPlotsFixture + const { plots, nbItemsPerRow, height } = customPlotsFixture return { checkpoint: { colors: { @@ -143,10 +143,13 @@ export const getExpectedCheckpointPlotsData = ( nbItemsPerRow, plots: plots.map(plot => ({ id: plot.id, - title: plot.yTitle, - values: plot.values.filter(values => domain.includes(values.group)) - })), - selectedMetrics + type: plot.type, + values: + plot.type === CustomPlotType.CHECKPOINT + ? plot.values.filter(value => domain.includes(value.group)) + : plot.values, + yTitle: plot.yTitle + })) } } } diff --git a/webview/src/plots/components/App.test.tsx b/webview/src/plots/components/App.test.tsx index e4f0bb3eec..9a42a8c9b2 100644 --- a/webview/src/plots/components/App.test.tsx +++ b/webview/src/plots/components/App.test.tsx @@ -13,14 +13,12 @@ import { } from '@testing-library/react' import '@testing-library/jest-dom/extend-expect' import comparisonTableFixture from 'dvc/src/test/fixtures/plotsDiff/comparison' -import checkpointPlotsFixture from 'dvc/src/test/fixtures/expShow/base/checkpointPlots' import customPlotsFixture from 'dvc/src/test/fixtures/expShow/base/customPlots' import plotsRevisionsFixture from 'dvc/src/test/fixtures/plotsDiff/revisions' import templatePlotsFixture from 'dvc/src/test/fixtures/plotsDiff/template/webview' import smoothTemplatePlotContent from 'dvc/src/test/fixtures/plotsDiff/template/smoothTemplatePlot' import manyTemplatePlots from 'dvc/src/test/fixtures/plotsDiff/template/virtualization' import { - CheckpointPlotsData, DEFAULT_SECTION_COLLAPSED, PlotsData, PlotNumberOfItemsPerRow, @@ -29,13 +27,13 @@ import { Section, TemplatePlotGroup, TemplatePlotsData, - CustomPlotType + CustomPlotType, + CustomPlotsData } from 'dvc/src/plots/webview/contract' import { MessageFromWebviewType, MessageToWebviewType } from 'dvc/src/webview/contract' -import { reorderObjectList } from 'dvc/src/util/array' import { act } from 'react-dom/test-utils' import { EXPERIMENT_WORKSPACE_ID } from 'dvc/src/cli/dvc/contract' import { VisualizationSpec } from 'react-vega' @@ -43,7 +41,7 @@ import { App } from './App' import { NewSectionBlock } from './templatePlots/TemplatePlots' import { SectionDescription } from './PlotsContainer' import { - CheckpointPlotsById, + CustomPlotsById, plotDataStore, TemplatePlotsById } from './plotDataStore' @@ -108,13 +106,6 @@ const originalOffsetWidth = Object.getOwnPropertyDescriptor( )?.value describe('App', () => { - const sectionPosition = { - [Section.CHECKPOINT_PLOTS]: 2, - [Section.TEMPLATE_PLOTS]: 0, - [Section.COMPARISON_TABLE]: 1, - [Section.CUSTOM_PLOTS]: 3 - } - const sendSetDataMessage = (data: PlotsData) => { const message = new MessageEvent('message', { data: { @@ -161,13 +152,6 @@ describe('App', () => { ] } as TemplatePlotsData - const getCheckpointMenuItem = (position: number) => - within( - screen.getAllByTestId('plots-container')[ - sectionPosition[Section.CHECKPOINT_PLOTS] - ] - ).getAllByTestId('icon-menu-item')[position] - const renderAppAndChangeSize = async ( data: PlotsData, nbItemsPerRow: number, @@ -180,11 +164,11 @@ describe('App', () => { ...data, sectionCollapsed: DEFAULT_SECTION_COLLAPSED } - if (section === Section.CHECKPOINT_PLOTS) { - plotsData.checkpoint = { - ...data?.checkpoint, + if (section === Section.CUSTOM_PLOTS) { + plotsData.custom = { + ...data?.custom, ...withSize - } as CheckpointPlotsData + } as CustomPlotsData } if (section === Section.TEMPLATE_PLOTS) { plotsData.template = { @@ -215,7 +199,7 @@ describe('App', () => { jest .spyOn(HTMLElement.prototype, 'clientHeight', 'get') .mockImplementation(() => heightToSuppressVegaError) - plotDataStore[Section.CHECKPOINT_PLOTS] = {} as CheckpointPlotsById + plotDataStore[Section.CUSTOM_PLOTS] = {} as CustomPlotsById plotDataStore[Section.TEMPLATE_PLOTS] = {} as TemplatePlotsById }) @@ -250,9 +234,7 @@ describe('App', () => { }) it('should render the empty state when given data with no plots', async () => { - renderAppWithOptionalData({ - checkpoint: null - }) + renderAppWithOptionalData({ custom: null }) const emptyState = await screen.findByText('No Plots Detected.') expect(emptyState).toBeInTheDocument() @@ -260,7 +242,6 @@ describe('App', () => { it('should render loading section states when given a single revision which has not been fetched', async () => { renderAppWithOptionalData({ - checkpoint: null, comparison: { height: undefined, nbItemsPerRow: PlotNumberOfItemsPerRow.TWO, @@ -298,12 +279,11 @@ describe('App', () => { }) const loading = await screen.findAllByText('Loading...') - expect(loading).toHaveLength(3) + expect(loading).toHaveLength(2) }) it('should render the Add Plots and Add Experiments get started button when there are experiments which have plots that are all unselected', async () => { renderAppWithOptionalData({ - checkpoint: null, hasPlots: true, hasUnselectedPlots: true, selectedRevisions: [{} as Revision] @@ -334,7 +314,7 @@ describe('App', () => { it('should render only the Add Experiments get started button when no experiments are selected', async () => { renderAppWithOptionalData({ - checkpoint: null, + custom: null, hasPlots: true, hasUnselectedPlots: false, selectedRevisions: undefined @@ -353,30 +333,6 @@ describe('App', () => { }) }) - it('should render other sections given a message with only checkpoint plots data', () => { - renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture - }) - - expect(screen.queryByText('Loading Plots...')).not.toBeInTheDocument() - expect(screen.getByText('Trends')).toBeInTheDocument() - expect(screen.getByText('Data Series')).toBeInTheDocument() - expect(screen.getByText('Images')).toBeInTheDocument() - expect(screen.getByText('Custom')).toBeInTheDocument() - expect(screen.getAllByText('No Plots to Display')).toHaveLength(2) - expect(screen.getByText('No Images to Compare')).toBeInTheDocument() - }) - - it('should render checkpoint even when there is no checkpoint plots data', () => { - renderAppWithOptionalData({ - template: templatePlotsFixture - }) - - expect(screen.queryByText('Loading Plots...')).not.toBeInTheDocument() - expect(screen.getByText('Trends')).toBeInTheDocument() - expect(screen.getAllByText('No Plots to Display')).toHaveLength(2) - }) - it('should render an empty state given a message with only custom plots data', () => { renderAppWithOptionalData({ custom: customPlotsFixture @@ -390,12 +346,13 @@ describe('App', () => { it('should render custom with "No Plots to Display" message when there is no custom plots data', () => { renderAppWithOptionalData({ - comparison: comparisonTableFixture + comparison: comparisonTableFixture, + template: templatePlotsFixture }) expect(screen.queryByText('Loading Plots...')).not.toBeInTheDocument() expect(screen.getByText('Custom')).toBeInTheDocument() - expect(screen.getAllByText('No Plots to Display')).toHaveLength(3) + expect(screen.getByText('No Plots to Display')).toBeInTheDocument() }) it('should render custom with "No Plots Added" message when there are no plots added', () => { @@ -409,7 +366,7 @@ describe('App', () => { expect(screen.queryByText('Loading Plots...')).not.toBeInTheDocument() expect(screen.getByText('Custom')).toBeInTheDocument() - expect(screen.getAllByText('No Plots to Display')).toHaveLength(2) + expect(screen.getByText('No Plots to Display')).toBeInTheDocument() expect(screen.getByText('No Plots Added')).toBeInTheDocument() }) @@ -417,7 +374,7 @@ describe('App', () => { const expectedSectionName = 'Images' renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + custom: customPlotsFixture }) sendSetDataMessage({ @@ -431,8 +388,8 @@ describe('App', () => { const emptyStateText = 'No Images to Compare' renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture, - comparison: comparisonTableFixture + comparison: comparisonTableFixture, + template: templatePlotsFixture }) expect(screen.queryByText(emptyStateText)).not.toBeInTheDocument() @@ -446,11 +403,10 @@ describe('App', () => { expect(emptyState).toBeInTheDocument() }) - it('should remove checkpoint plots given a message showing checkpoint plots as null', async () => { + it('should remove checkpoint plots given a message showing custom plots as null', async () => { const emptyStateText = 'No Plots to Display' renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture, comparison: comparisonTableFixture, custom: customPlotsFixture, template: templatePlotsFixture @@ -459,7 +415,7 @@ describe('App', () => { expect(screen.queryByText(emptyStateText)).not.toBeInTheDocument() sendSetDataMessage({ - checkpoint: null + custom: null }) const emptyState = await screen.findByText(emptyStateText) @@ -469,24 +425,25 @@ describe('App', () => { it('should remove all sections from the document if there is no data provided', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture }) - expect(screen.getByText('Trends')).toBeInTheDocument() + expect(screen.getByText('Images')).toBeInTheDocument() sendSetDataMessage({ - checkpoint: null + comparison: null }) - expect(screen.queryByText('Trends')).not.toBeInTheDocument() + expect(screen.queryByText('Images')).not.toBeInTheDocument() }) - it('should toggle the checkpoint plots section in state when its header is clicked', async () => { + it('should toggle the custom plots section in state when its header is clicked', async () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) - const summaryElement = await screen.findByText('Trends') + const summaryElement = await screen.findByText('Custom') const visiblePlots = await screen.findAllByLabelText('Vega visualization') for (const visiblePlot of visiblePlots) { expect(visiblePlot).toBeInTheDocument() @@ -499,14 +456,14 @@ describe('App', () => { }) expect(mockPostMessage).toHaveBeenCalledWith({ - payload: { [Section.CHECKPOINT_PLOTS]: true }, + payload: { [Section.CUSTOM_PLOTS]: true }, type: MessageFromWebviewType.TOGGLE_PLOTS_SECTION }) sendSetDataMessage({ sectionCollapsed: { ...DEFAULT_SECTION_COLLAPSED, - [Section.CHECKPOINT_PLOTS]: true + [Section.CUSTOM_PLOTS]: true } }) @@ -515,21 +472,22 @@ describe('App', () => { ).not.toBeInTheDocument() }) - it('should not toggle the checkpoint plots section when its header is clicked and its title is selected', async () => { + it('should not toggle the custom plots section when its header is clicked and its title is selected', async () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) - const summaryElement = await screen.findByText('Trends') + const summaryElement = await screen.findByText('Custom') - createWindowTextSelection('Trends', 2) + createWindowTextSelection('Custom', 2) fireEvent.click(summaryElement, { bubbles: true, cancelable: true }) expect(mockPostMessage).not.toHaveBeenCalledWith({ - payload: { [Section.CHECKPOINT_PLOTS]: true }, + payload: { [Section.CUSTOM_PLOTS]: true }, type: MessageFromWebviewType.TOGGLE_PLOTS_SECTION }) @@ -540,25 +498,25 @@ describe('App', () => { }) expect(mockPostMessage).toHaveBeenCalledWith({ - payload: { [Section.CHECKPOINT_PLOTS]: true }, + payload: { [Section.CUSTOM_PLOTS]: true }, type: MessageFromWebviewType.TOGGLE_PLOTS_SECTION }) }) - it('should not toggle the checkpoint plots section if the tooltip is clicked', () => { + it('should not toggle the comparison plots section if the tooltip is clicked', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture }) - const checkpointsTooltipToggle = screen.getAllByTestId( + const comparisonTooltipToggle = screen.getAllByTestId( 'info-tooltip-toggle' - )[2] - fireEvent.mouseEnter(checkpointsTooltipToggle, { + )[1] + fireEvent.mouseEnter(comparisonTooltipToggle, { bubbles: true, cancelable: true }) - const tooltip = screen.getByTestId('tooltip-checkpoint-plots') + const tooltip = screen.getByTestId('tooltip-comparison-plots') const tooltipLink = within(tooltip).getByRole('link') fireEvent.click(tooltipLink, { bubbles: true, @@ -566,30 +524,31 @@ describe('App', () => { }) expect(mockPostMessage).not.toHaveBeenCalledWith({ - payload: { [Section.CHECKPOINT_PLOTS]: true }, + payload: { [Section.CUSTOM_PLOTS]: true }, type: MessageFromWebviewType.TOGGLE_PLOTS_SECTION }) - fireEvent.click(checkpointsTooltipToggle, { + fireEvent.click(comparisonTooltipToggle, { bubbles: true, cancelable: true }) expect(mockPostMessage).not.toHaveBeenCalledWith({ - payload: { [Section.CHECKPOINT_PLOTS]: true }, + payload: { [Section.CUSTOM_PLOTS]: true }, type: MessageFromWebviewType.TOGGLE_PLOTS_SECTION }) }) - it('should not toggle the checkpoint plots section when its header is clicked and the content of its tooltip is selected', async () => { + it('should not toggle the custom plots section when its header is clicked and the content of its tooltip is selected', async () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) - const summaryElement = await screen.findByText('Trends') + const summaryElement = await screen.findByText('Custom') createWindowTextSelection( // eslint-disable-next-line testing-library/no-node-access - SectionDescription['checkpoint-plots'].props.children, + SectionDescription['custom-plots'].props.children, 2 ) fireEvent.click(summaryElement, { @@ -598,7 +557,7 @@ describe('App', () => { }) expect(mockPostMessage).not.toHaveBeenCalledWith({ - payload: { [Section.CHECKPOINT_PLOTS]: true }, + payload: { [Section.CUSTOM_PLOTS]: true }, type: MessageFromWebviewType.TOGGLE_PLOTS_SECTION }) @@ -609,96 +568,15 @@ describe('App', () => { }) expect(mockPostMessage).toHaveBeenCalledWith({ - payload: { [Section.CHECKPOINT_PLOTS]: true }, + payload: { [Section.CUSTOM_PLOTS]: true }, type: MessageFromWebviewType.TOGGLE_PLOTS_SECTION }) }) - it('should toggle the visibility of plots when clicking the metrics in the metrics picker', async () => { - renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture - }) - - const summaryElement = await screen.findByText('Trends') - fireEvent.click(summaryElement, { - bubbles: true, - cancelable: true - }) - - expect(screen.getByTestId('plot-summary.json:loss')).toBeInTheDocument() - - const pickerButton = getCheckpointMenuItem(0) - fireEvent.mouseEnter(pickerButton) - fireEvent.click(pickerButton) - - const lossItem = await screen.findByText('summary.json:loss', { - ignore: 'text' - }) - - fireEvent.click(lossItem, { - bubbles: true, - cancelable: true - }) - - expect( - screen.queryByTestId('plot-summary.json:loss') - ).not.toBeInTheDocument() - - fireEvent.mouseEnter(pickerButton) - fireEvent.click(pickerButton) - - fireEvent.click(lossItem, { - bubbles: true, - cancelable: true - }) - - expect(screen.getByTestId('plot-summary.json:loss')).toBeInTheDocument() - }) - - it('should send a message to the extension with the selected metrics when toggling the visibility of a plot', async () => { - renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture - }) - - const pickerButton = getCheckpointMenuItem(0) - fireEvent.mouseEnter(pickerButton) - fireEvent.click(pickerButton) - - const lossItem = await screen.findByText('summary.json:loss') - - fireEvent.click(lossItem, { - bubbles: true, - cancelable: true - }) - - expect(mockPostMessage).toHaveBeenCalledWith({ - payload: [ - 'summary.json:accuracy', - 'summary.json:val_accuracy', - 'summary.json:val_loss' - ], - type: MessageFromWebviewType.TOGGLE_METRIC - }) - - fireEvent.click(lossItem, { - bubbles: true, - cancelable: true - }) - - expect(mockPostMessage).toHaveBeenCalledWith({ - payload: [ - 'summary.json:accuracy', - 'summary.json:loss', - 'summary.json:val_accuracy', - 'summary.json:val_loss' - ], - type: MessageFromWebviewType.TOGGLE_METRIC - }) - }) - it('should send a message to the extension with the selected size when changing the size of plots', () => { const store = renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) const plotResizer = screen.getAllByTestId('vertical-plot-resizer')[0] @@ -708,7 +586,7 @@ describe('App', () => { expect(mockPostMessage).toHaveBeenCalledWith({ payload: { nbItemsPerRow: PlotNumberOfItemsPerRow.ONE, - section: Section.CHECKPOINT_PLOTS + section: Section.CUSTOM_PLOTS }, type: MessageFromWebviewType.RESIZE_PLOTS }) @@ -718,7 +596,7 @@ describe('App', () => { expect(mockPostMessage).toHaveBeenCalledWith({ payload: { nbItemsPerRow: PlotNumberOfItemsPerRow.TWO, - section: Section.CHECKPOINT_PLOTS + section: Section.CUSTOM_PLOTS }, type: MessageFromWebviewType.RESIZE_PLOTS }) @@ -730,7 +608,7 @@ describe('App', () => { expect(mockPostMessage).toHaveBeenCalledWith({ payload: { nbItemsPerRow: PlotNumberOfItemsPerRow.THREE, - section: Section.CHECKPOINT_PLOTS + section: Section.CUSTOM_PLOTS }, type: MessageFromWebviewType.RESIZE_PLOTS }) @@ -740,7 +618,7 @@ describe('App', () => { expect(mockPostMessage).toHaveBeenCalledWith({ payload: { nbItemsPerRow: PlotNumberOfItemsPerRow.FOUR, - section: Section.CHECKPOINT_PLOTS + section: Section.CUSTOM_PLOTS }, type: MessageFromWebviewType.RESIZE_PLOTS }) @@ -748,7 +626,8 @@ describe('App', () => { it('should not send a message to the extension with the selected size when changing the size of plots and pressing escape', () => { const store = renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) const plotResizer = screen.getAllByTestId('vertical-plot-resizer')[0] @@ -758,24 +637,25 @@ describe('App', () => { expect(mockPostMessage).not.toHaveBeenCalledWith({ payload: { nbItemsPerRow: PlotNumberOfItemsPerRow.ONE, - section: Section.CHECKPOINT_PLOTS + section: Section.CUSTOM_PLOTS }, type: MessageFromWebviewType.RESIZE_PLOTS }) }) - it('should display the checkpoint plots in the order stored', () => { + it('should display the custom plots in the order stored', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) let plots = screen.getAllByTestId(/summary\.json/) expect(plots.map(plot => plot.id)).toStrictEqual([ - 'summary.json:loss', - 'summary.json:accuracy', - 'summary.json:val_loss', - 'summary.json:val_accuracy' + 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss', + 'custom-summary.json:accuracy' ]) dragAndDrop(plots[1], plots[0]) @@ -783,24 +663,25 @@ describe('App', () => { plots = screen.getAllByTestId(/summary\.json/) expect(plots.map(plot => plot.id)).toStrictEqual([ - 'summary.json:accuracy', - 'summary.json:loss', - 'summary.json:val_loss', - 'summary.json:val_accuracy' + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-summary.json:loss', + 'custom-summary.json:accuracy' ]) }) - it('should send a message to the extension when the checkpoint plots are reordered', () => { + it('should send a message to the extension when the custom plots are reordered', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) const plots = screen.getAllByTestId(/summary\.json/) expect(plots.map(plot => plot.id)).toStrictEqual([ - 'summary.json:loss', - 'summary.json:accuracy', - 'summary.json:val_loss', - 'summary.json:val_accuracy' + 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss', + 'custom-summary.json:accuracy' ]) mockPostMessage.mockClear() @@ -808,57 +689,37 @@ describe('App', () => { dragAndDrop(plots[2], plots[0]) const expectedOrder = [ - 'summary.json:val_loss', - 'summary.json:loss', - 'summary.json:accuracy', - 'summary.json:val_accuracy' + 'custom-summary.json:loss', + 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:accuracy' ] expect(mockPostMessage).toHaveBeenCalledTimes(1) expect(mockPostMessage).toHaveBeenCalledWith({ payload: expectedOrder, - type: MessageFromWebviewType.REORDER_PLOTS_METRICS + type: MessageFromWebviewType.REORDER_PLOTS_CUSTOM }) expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual(expectedOrder) }) - it('should remove the checkpoint plot from the order if it is removed from the plots', () => { - renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture - }) - - let plots = screen.getAllByTestId(/summary\.json/) - dragAndDrop(plots[1], plots[0]) - - sendSetDataMessage({ - checkpoint: { - ...checkpointPlotsFixture, - plots: checkpointPlotsFixture.plots.slice(1) - } - }) - plots = screen.getAllByTestId(/summary\.json/) - expect(plots.map(plot => plot.id)).toStrictEqual([ - 'summary.json:accuracy', - 'summary.json:val_loss', - 'summary.json:val_accuracy' - ]) - }) - it('should add a custom plot if a user creates a custom plot', () => { renderAppWithOptionalData({ comparison: comparisonTableFixture, custom: { ...customPlotsFixture, - plots: customPlotsFixture.plots.slice(1) + plots: customPlotsFixture.plots.slice(0, 3) } }) expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' + 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss' ]) sendSetDataMessage({ @@ -868,8 +729,10 @@ describe('App', () => { expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ + 'custom-metrics:summary.json:loss-params:params.yaml:dropout', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-metrics:summary.json:loss-params:params.yaml:dropout' + 'custom-summary.json:loss', + 'custom-summary.json:accuracy' ]) }) @@ -883,7 +746,9 @@ describe('App', () => { screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss', + 'custom-summary.json:accuracy' ]) sendSetDataMessage({ @@ -896,87 +761,28 @@ describe('App', () => { expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss', + 'custom-summary.json:accuracy' ]) }) - it('should not change the metric order in the hover menu by reordering the plots', () => { - renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture - }) - - const [pickerButton] = within( - screen.getAllByTestId('plots-container')[ - sectionPosition[Section.CHECKPOINT_PLOTS] - ] - ).queryAllByTestId('icon-menu-item') - - fireEvent.mouseEnter(pickerButton) - fireEvent.click(pickerButton) - - let options = screen.getAllByTestId('select-menu-option-label') - const optionsOrder = [ - 'summary.json:accuracy', - 'summary.json:loss', - 'summary.json:val_accuracy', - 'summary.json:val_loss' - ] - expect(options.map(({ textContent }) => textContent)).toStrictEqual( - optionsOrder - ) - - fireEvent.click(pickerButton) - - let plots = screen.getAllByTestId(/summary\.json/) - const newPlotOrder = [ - 'summary.json:val_accuracy', - 'summary.json:loss', - 'summary.json:accuracy', - 'summary.json:val_loss' - ] - expect(plots.map(plot => plot.id)).not.toStrictEqual(newPlotOrder) - - dragAndDrop(plots[3], plots[0]) - sendSetDataMessage({ - checkpoint: { - ...checkpointPlotsFixture, - plots: reorderObjectList( - newPlotOrder, - checkpointPlotsFixture.plots, - 'title' - ) - } - }) - - plots = screen.getAllByTestId(/summary\.json/) - - expect(plots.map(plot => plot.id)).toStrictEqual(newPlotOrder) - - fireEvent.mouseEnter(pickerButton) - fireEvent.click(pickerButton) - - options = screen.getAllByTestId('select-menu-option-label') - expect(options.map(({ textContent }) => textContent)).toStrictEqual( - optionsOrder - ) - }) - it('should not be possible to drag a plot from a section to another', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture, + custom: customPlotsFixture, template: templatePlotsFixture }) - const checkpointPlots = screen.getAllByTestId(/summary\.json/) + const customPlots = screen.getAllByTestId(/summary\.json/) const templatePlots = screen.getAllByTestId(/^plot_/) - dragAndDrop(templatePlots[0], checkpointPlots[2]) + dragAndDrop(templatePlots[0], customPlots[2]) - expect(checkpointPlots.map(plot => plot.id)).toStrictEqual([ - 'summary.json:loss', - 'summary.json:accuracy', - 'summary.json:val_loss', - 'summary.json:val_accuracy' + expect(customPlots.map(plot => plot.id)).toStrictEqual([ + 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss', + 'custom-summary.json:accuracy' ]) }) @@ -1193,32 +999,34 @@ describe('App', () => { ]) }) - it('should show a drop target at the end of the checkpoint plots when moving a plot inside the section but not over any other plot', () => { + it('should show a drop target at the end of the custom plots when moving a plot inside the section but not over any other plot', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + custom: customPlotsFixture, + template: templatePlotsFixture }) const plots = screen.getAllByTestId(/summary\.json/) - dragEnter(plots[0], 'checkpoint-plots', DragEnterDirection.LEFT) + dragEnter(plots[0], 'custom-plots', DragEnterDirection.LEFT) expect(screen.getByTestId('plot_drop-target')).toBeInTheDocument() }) - it('should show a drop a plot at the end of the checkpoint plots when moving a plot inside the section but not over any other plot', () => { + it('should show a drop a plot at the end of the custom plots when moving a plot inside the section but not over any other plot', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + custom: customPlotsFixture, + template: templatePlotsFixture }) const plots = screen.getAllByTestId(/summary\.json/) - dragAndDrop(plots[0], screen.getByTestId('checkpoint-plots')) + dragAndDrop(plots[0], screen.getByTestId('custom-plots')) const expectedOrder = [ - 'summary.json:accuracy', - 'summary.json:val_loss', - 'summary.json:val_accuracy', - 'summary.json:loss' + 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss', + 'custom-summary.json:accuracy', + 'custom-metrics:summary.json:loss-params:params.yaml:dropout' ] expect( @@ -1413,7 +1221,8 @@ describe('App', () => { it('should open a modal with the plot zoomed in when clicking a checkpoint plot', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture + comparison: comparisonTableFixture, + custom: customPlotsFixture }) expect(screen.queryByTestId('modal')).not.toBeInTheDocument() @@ -1475,14 +1284,14 @@ describe('App', () => { it('should show a tooltip with the meaning of each plot section', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture, comparison: comparisonTableFixture, custom: customPlotsFixture, template: complexTemplatePlotsFixture }) - const [templateInfo, comparisonInfo, checkpointInfo, customInfo] = - screen.getAllByTestId('info-tooltip-toggle') + const [templateInfo, comparisonInfo, customInfo] = screen.getAllByTestId( + 'info-tooltip-toggle' + ) fireEvent.mouseEnter(templateInfo, { bubbles: true }) expect(screen.getByTestId('tooltip-template-plots')).toBeInTheDocument() @@ -1490,16 +1299,12 @@ describe('App', () => { fireEvent.mouseEnter(comparisonInfo, { bubbles: true }) expect(screen.getByTestId('tooltip-comparison-plots')).toBeInTheDocument() - fireEvent.mouseEnter(checkpointInfo, { bubbles: true }) - expect(screen.getByTestId('tooltip-checkpoint-plots')).toBeInTheDocument() - fireEvent.mouseEnter(customInfo, { bubbles: true }) expect(screen.getByTestId('tooltip-custom-plots')).toBeInTheDocument() }) it('should dismiss a tooltip by pressing esc', () => { renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture, comparison: comparisonTableFixture, custom: customPlotsFixture, template: complexTemplatePlotsFixture @@ -1520,7 +1325,7 @@ describe('App', () => { }) describe('Virtualization', () => { - const createCheckpointPlots = (nbOfPlots: number): CheckpointPlotsData => { + const createCustomPlots = (nbOfPlots: number): CustomPlotsData => { const plots = [] for (let i = 0; i < nbOfPlots; i++) { const id = `plot-${i}` @@ -1533,10 +1338,10 @@ describe('App', () => { }) } return { - ...checkpointPlotsFixture, + ...customPlotsFixture, plots, selectedMetrics: plots.map(plot => plot.id) - } as CheckpointPlotsData + } as CustomPlotsData } const resizeScreen = (width: number, store: typeof plotsStore) => { @@ -1550,17 +1355,17 @@ describe('App', () => { } describe('Large plots', () => { - it('should wrap the checkpoint plots in a big grid (virtualize them) when there are more than ten large plots', async () => { + it('should wrap the custom plots in a big grid (virtualize them) when there are more than ten large plots', async () => { await renderAppAndChangeSize( - { checkpoint: createCheckpointPlots(11) }, + { comparison: comparisonTableFixture, custom: createCustomPlots(11) }, PlotNumberOfItemsPerRow.ONE, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) expect(screen.getByRole('grid')).toBeInTheDocument() sendSetDataMessage({ - checkpoint: createCheckpointPlots(50) + custom: createCustomPlots(50) }) await screen.findAllByTestId('plots-wrapper') @@ -1568,17 +1373,17 @@ describe('App', () => { expect(screen.getByRole('grid')).toBeInTheDocument() }) - it('should not wrap the checkpoint plots in a big grid (virtualize them) when there are ten or fewer large plots', async () => { + it('should not wrap the custom plots in a big grid (virtualize them) when there are ten or fewer large plots', async () => { await renderAppAndChangeSize( - { checkpoint: createCheckpointPlots(10) }, + { comparison: comparisonTableFixture, custom: createCustomPlots(10) }, PlotNumberOfItemsPerRow.ONE, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) expect(screen.queryByRole('grid')).not.toBeInTheDocument() sendSetDataMessage({ - checkpoint: createCheckpointPlots(1) + custom: createCustomPlots(1) }) await screen.findAllByTestId('plots-wrapper') @@ -1623,30 +1428,28 @@ describe('App', () => { }) describe('Sizing', () => { - const checkpoint = createCheckpointPlots(25) + const custom = createCustomPlots(25) let store: typeof plotsStore beforeEach(async () => { store = await renderAppAndChangeSize( - { checkpoint }, + { comparison: comparisonTableFixture, custom }, PlotNumberOfItemsPerRow.ONE, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) }) it('should render the plots correctly when the screen is larger than 2000px', () => { - resizeScreen(3000, store) - let plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) + expect(plots[4].id).toBe(custom.plots[4].yTitle) expect(plots.length).toBe(OVERSCAN_ROW_COUNT + 1) resizeScreen(5453, store) plots = screen.getAllByTestId(/^plot-/) - expect(plots[3].id).toBe(checkpoint.plots[3].yTitle) + expect(plots[3].id).toBe(custom.plots[3].yTitle) expect(plots.length).toBe(OVERSCAN_ROW_COUNT + 1) }) @@ -1655,7 +1458,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[12].id).toBe(checkpoint.plots[12].yTitle) + expect(plots[12].id).toBe(custom.plots[12].yTitle) expect(plots.length).toBe(OVERSCAN_ROW_COUNT + 1) }) @@ -1664,7 +1467,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[14].id).toBe(checkpoint.plots[14].yTitle) + expect(plots[14].id).toBe(custom.plots[14].yTitle) expect(plots.length).toBe(1 + OVERSCAN_ROW_COUNT) // Only the first and the next lines defined by the overscan row count will be rendered }) @@ -1673,7 +1476,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) + expect(plots[4].id).toBe(custom.plots[4].yTitle) }) }) }) @@ -1681,9 +1484,9 @@ describe('App', () => { describe('Regular plots', () => { it('should wrap the checkpoint plots in a big grid (virtualize them) when there are more than fifteen regular plots', async () => { await renderAppAndChangeSize( - { checkpoint: createCheckpointPlots(16) }, + { comparison: comparisonTableFixture, custom: createCustomPlots(16) }, PlotNumberOfItemsPerRow.TWO, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) expect(screen.getByRole('grid')).toBeInTheDocument() @@ -1691,9 +1494,9 @@ describe('App', () => { it('should not wrap the checkpoint plots in a big grid (virtualize them) when there are eight or fifteen regular plots', async () => { await renderAppAndChangeSize( - { checkpoint: createCheckpointPlots(15) }, + { comparison: comparisonTableFixture, custom: createCustomPlots(15) }, PlotNumberOfItemsPerRow.TWO, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) expect(screen.queryByRole('grid')).not.toBeInTheDocument() @@ -1720,14 +1523,14 @@ describe('App', () => { }) describe('Sizing', () => { - const checkpoint = createCheckpointPlots(25) + const custom = createCustomPlots(25) let store: typeof plotsStore beforeEach(async () => { store = await renderAppAndChangeSize( - { checkpoint }, + { comparison: comparisonTableFixture, custom }, PlotNumberOfItemsPerRow.TWO, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) }) @@ -1736,15 +1539,15 @@ describe('App', () => { let plots = screen.getAllByTestId(/^plot-/) - expect(plots[20].id).toBe(checkpoint.plots[20].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[20].id).toBe(custom.plots[20].yTitle) + expect(plots.length).toBe(custom.plots.length) resizeScreen(6453, store) plots = screen.getAllByTestId(/^plot-/) - expect(plots[19].id).toBe(checkpoint.plots[19].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[19].id).toBe(custom.plots[19].yTitle) + expect(plots.length).toBe(custom.plots.length) }) it('should render the plots correctly when the screen is larger than 1600px (but less than 2000px)', () => { @@ -1752,8 +1555,8 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[7].id).toBe(checkpoint.plots[7].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[7].id).toBe(custom.plots[7].yTitle) + expect(plots.length).toBe(custom.plots.length) }) it('should render the plots correctly when the screen is larger than 800px (but less than 1600px)', () => { @@ -1761,8 +1564,8 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[7].id).toBe(checkpoint.plots[7].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[7].id).toBe(custom.plots[7].yTitle) + expect(plots.length).toBe(custom.plots.length) }) it('should render the plots correctly when the screen is smaller than 800px', () => { @@ -1770,7 +1573,7 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) + expect(plots[4].id).toBe(custom.plots[4].yTitle) }) }) }) @@ -1778,9 +1581,9 @@ describe('App', () => { describe('Smaller plots', () => { it('should wrap the checkpoint plots in a big grid (virtualize them) when there are more than twenty small plots', async () => { await renderAppAndChangeSize( - { checkpoint: createCheckpointPlots(21) }, + { comparison: comparisonTableFixture, custom: createCustomPlots(21) }, PlotNumberOfItemsPerRow.FOUR, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) expect(screen.getByRole('grid')).toBeInTheDocument() @@ -1788,9 +1591,9 @@ describe('App', () => { it('should not wrap the checkpoint plots in a big grid (virtualize them) when there are twenty or fewer small plots', async () => { await renderAppAndChangeSize( - { checkpoint: createCheckpointPlots(20) }, + { comparison: comparisonTableFixture, custom: createCustomPlots(20) }, PlotNumberOfItemsPerRow.FOUR, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) expect(screen.queryByRole('grid')).not.toBeInTheDocument() @@ -1817,14 +1620,14 @@ describe('App', () => { }) describe('Sizing', () => { - const checkpoint = createCheckpointPlots(25) + const custom = createCustomPlots(25) let store: typeof plotsStore beforeEach(async () => { store = await renderAppAndChangeSize( - { checkpoint }, + { comparison: comparisonTableFixture, custom }, PlotNumberOfItemsPerRow.FOUR, - Section.CHECKPOINT_PLOTS + Section.CUSTOM_PLOTS ) }) @@ -1833,15 +1636,15 @@ describe('App', () => { let plots = screen.getAllByTestId(/^plot-/) - expect(plots[7].id).toBe(checkpoint.plots[7].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[7].id).toBe(custom.plots[7].yTitle) + expect(plots.length).toBe(custom.plots.length) resizeScreen(5473, store) plots = screen.getAllByTestId(/^plot-/) - expect(plots[9].id).toBe(checkpoint.plots[9].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[9].id).toBe(custom.plots[9].yTitle) + expect(plots.length).toBe(custom.plots.length) }) it('should render the plots correctly when the screen is larger than 1600px (but less than 2000px)', () => { @@ -1849,8 +1652,8 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[24].id).toBe(checkpoint.plots[24].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[24].id).toBe(custom.plots[24].yTitle) + expect(plots.length).toBe(custom.plots.length) }) it('should render the plots correctly when the screen is larger than 800px (but less than 1600px)', () => { @@ -1858,8 +1661,8 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[9].id).toBe(checkpoint.plots[9].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[9].id).toBe(custom.plots[9].yTitle) + expect(plots.length).toBe(custom.plots.length) }) it('should render the plots correctly when the screen is smaller than 800px but larger than 600px', () => { @@ -1867,8 +1670,8 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[9].id).toBe(checkpoint.plots[9].yTitle) - expect(plots.length).toBe(checkpoint.plots.length) + expect(plots[9].id).toBe(custom.plots[9].yTitle) + expect(plots.length).toBe(custom.plots.length) }) it('should render the plots correctly when the screen is smaller than 600px', () => { @@ -1876,33 +1679,12 @@ describe('App', () => { const plots = screen.getAllByTestId(/^plot-/) - expect(plots[4].id).toBe(checkpoint.plots[4].yTitle) + expect(plots[4].id).toBe(custom.plots[4].yTitle) }) }) }) }) - describe('Context Menu Suppression', () => { - it('Suppresses the context menu with no plots data', () => { - renderAppWithOptionalData() - const target = screen.getByText('Loading Plots...') - const contextMenuEvent = createEvent.contextMenu(target) - fireEvent(target, contextMenuEvent) - expect(contextMenuEvent.defaultPrevented).toBe(true) - }) - - it('Suppresses the context menu with plots data', () => { - renderAppWithOptionalData({ - checkpoint: checkpointPlotsFixture, - sectionCollapsed: DEFAULT_SECTION_COLLAPSED - }) - const target = screen.getByText('Trends') - const contextMenuEvent = createEvent.contextMenu(target) - fireEvent(target, contextMenuEvent) - expect(contextMenuEvent.defaultPrevented).toBe(true) - }) - }) - // eslint-disable-next-line sonarjs/cognitive-complexity describe('Ribbon', () => { const getDisplayedRevisionOrder = () => { diff --git a/webview/src/plots/components/PlotsContainer.tsx b/webview/src/plots/components/PlotsContainer.tsx index 6c775a725a..5e1f7ad9de 100644 --- a/webview/src/plots/components/PlotsContainer.tsx +++ b/webview/src/plots/components/PlotsContainer.tsx @@ -37,17 +37,6 @@ export interface PlotsContainerProps { } export const SectionDescription = { - // "Trends" - [Section.CHECKPOINT_PLOTS]: ( - - Automatically generated and updated linear plots that show metric value - per epoch if{' '} - - checkpoints - {' '} - are enabled. - - ), // "Custom" [Section.CUSTOM_PLOTS]: ( diff --git a/webview/src/plots/components/plotDataStore.ts b/webview/src/plots/components/plotDataStore.ts index 51436b9d96..9d62f88f9c 100644 --- a/webview/src/plots/components/plotDataStore.ts +++ b/webview/src/plots/components/plotDataStore.ts @@ -5,14 +5,12 @@ import { TemplatePlotEntry } from 'dvc/src/plots/webview/contract' -export type CheckpointPlotsById = { [key: string]: CheckpointPlotData } export type CustomPlotsById = { [key: string]: CustomPlotData } export type TemplatePlotsById = { [key: string]: TemplatePlotEntry } export const plotDataStore = { - [Section.CHECKPOINT_PLOTS]: {} as CheckpointPlotsById, [Section.TEMPLATE_PLOTS]: {} as TemplatePlotsById, - [Section.COMPARISON_TABLE]: {} as CheckpointPlotsById, // This category is unused but exists only to make typings easier, + [Section.COMPARISON_TABLE]: {} as CustomPlotsById, // This category is unused but exists only to make typings easier, [Section.CUSTOM_PLOTS]: {} as CustomPlotsById } diff --git a/webview/src/plots/hooks/useGetPlot.ts b/webview/src/plots/hooks/useGetPlot.ts index 69c8bd2c16..aafa6e1ce8 100644 --- a/webview/src/plots/hooks/useGetPlot.ts +++ b/webview/src/plots/hooks/useGetPlot.ts @@ -18,8 +18,7 @@ export const useGetPlot = ( id: string, spec?: VisualizationSpec ) => { - const isPlotWithSpec = - section === Section.CHECKPOINT_PLOTS || section === Section.CUSTOM_PLOTS + const isPlotWithSpec = section === Section.CUSTOM_PLOTS const storeSection = getStoreSection(section) const snapshot = useSelector( (state: PlotsState) => state[storeSection].plotsSnapshots diff --git a/webview/src/stories/Plots.stories.tsx b/webview/src/stories/Plots.stories.tsx index 5ad6e6b298..be2c2c2c48 100644 --- a/webview/src/stories/Plots.stories.tsx +++ b/webview/src/stories/Plots.stories.tsx @@ -11,7 +11,6 @@ import { PlotNumberOfItemsPerRow } from 'dvc/src/plots/webview/contract' import { MessageToWebviewType } from 'dvc/src/webview/contract' -import checkpointPlotsFixture from 'dvc/src/test/fixtures/expShow/base/checkpointPlots' import customPlotsFixture from 'dvc/src/test/fixtures/expShow/base/customPlots' import templatePlotsFixture from 'dvc/src/test/fixtures/plotsDiff/template' import manyTemplatePlots from 'dvc/src/test/fixtures/plotsDiff/template/virtualization' @@ -27,23 +26,18 @@ import '../shared/style.scss' import '../plots/components/styles.module.scss' import { feedStore } from '../plots/components/App' import { plotsReducers } from '../plots/store' - -const manyCheckpointPlots = ( - length: number, - size = PlotNumberOfItemsPerRow.TWO -) => - Array.from({ length }, () => checkpointPlotsFixture.plots[0]).map( - (plot, i) => { - const id = plot.id + i.toString() - return { - ...plot, - id, - title: truncateVerticalTitle(id, size) as string - } +// TBD review storybooks, making sure they all make sense +const manyCustomPlots = (length: number, size = PlotNumberOfItemsPerRow.TWO) => + Array.from({ length }, () => customPlotsFixture.plots[2]).map((plot, i) => { + const id = plot.id + i.toString() + return { + ...plot, + id, + yTitle: truncateVerticalTitle(id, size) as string } - ) + }) -const manyCheckpointPlotsFixture = manyCheckpointPlots(15) +const manyCustomPlotsFixture = manyCustomPlots(15) const MockedState: React.FC<{ data: PlotsData; children: React.ReactNode }> = ({ children, @@ -59,7 +53,6 @@ const MockedState: React.FC<{ data: PlotsData; children: React.ReactNode }> = ({ export default { args: { data: { - checkpoint: checkpointPlotsFixture, comparison: comparisonPlotsFixture, custom: customPlotsFixture, hasPlots: true, @@ -92,8 +85,8 @@ WithData.parameters = CHROMATIC_VIEWPORTS export const WithEmptyCheckpoints = Template.bind({}) WithEmptyCheckpoints.args = { data: { - checkpoint: { ...checkpointPlotsFixture, selectedMetrics: [] }, comparison: comparisonPlotsFixture, + custom: { ...customPlotsFixture }, sectionCollapsed: DEFAULT_SECTION_COLLAPSED, selectedRevisions: plotsRevisionsFixture, template: templatePlotsFixture @@ -104,7 +97,7 @@ WithEmptyCheckpoints.parameters = DISABLE_CHROMATIC_SNAPSHOTS export const WithCheckpointOnly = Template.bind({}) WithCheckpointOnly.args = { data: { - checkpoint: checkpointPlotsFixture, + custom: customPlotsFixture, sectionCollapsed: DEFAULT_SECTION_COLLAPSED, selectedRevisions: plotsRevisionsFixture } @@ -181,10 +174,6 @@ WithoutData.args = { export const AllLarge = Template.bind({}) AllLarge.args = { data: { - checkpoint: { - ...checkpointPlotsFixture, - nbItemsPerRow: PlotNumberOfItemsPerRow.ONE - }, comparison: { ...comparisonPlotsFixture, nbItemsPerRow: PlotNumberOfItemsPerRow.ONE @@ -227,13 +216,11 @@ AllSmall.parameters = CHROMATIC_VIEWPORTS export const VirtualizedPlots = Template.bind({}) VirtualizedPlots.args = { data: { - checkpoint: { - ...checkpointPlotsFixture, - plots: manyCheckpointPlotsFixture, - selectedMetrics: manyCheckpointPlotsFixture.map(plot => plot.id) - }, comparison: undefined, - custom: customPlotsFixture, + custom: { + ...customPlotsFixture, + plots: manyCustomPlotsFixture + }, sectionCollapsed: DEFAULT_SECTION_COLLAPSED, selectedRevisions: plotsRevisionsFixture, template: manyTemplatePlots(125) @@ -299,7 +286,6 @@ ScrolledHeaders.parameters = { export const ScrolledWithManyRevisions = Template.bind({}) ScrolledWithManyRevisions.args = { data: { - checkpoint: checkpointPlotsFixture, comparison: comparisonPlotsFixture, custom: customPlotsFixture, hasPlots: true, From 6216af08d6dd6801acf2da54805b1594e9720a74 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Wed, 8 Mar 2023 10:17:31 -0600 Subject: [PATCH 05/40] Fix vscode tests --- extension/src/plots/model/collect.test.ts | 43 ++++--------- extension/src/plots/model/index.ts | 15 ++--- extension/src/plots/webview/messages.ts | 62 +++++++++---------- .../test/fixtures/expShow/base/customPlots.ts | 47 ++++++++++---- .../test/suite/experiments/model/tree.test.ts | 19 +++--- extension/src/test/suite/plots/index.test.ts | 51 ++++++++++----- extension/src/test/suite/plots/util.ts | 13 ++-- 7 files changed, 138 insertions(+), 112 deletions(-) diff --git a/extension/src/plots/model/collect.test.ts b/extension/src/plots/model/collect.test.ts index 679402b3e3..6e99de325f 100644 --- a/extension/src/plots/model/collect.test.ts +++ b/extension/src/plots/model/collect.test.ts @@ -7,7 +7,9 @@ import { collectCustomPlotsData } from './collect' import plotsDiffFixture from '../../test/fixtures/plotsDiff/output' -import customPlotsFixture from '../../test/fixtures/expShow/base/customPlots' +import customPlotsFixture, { + customPlotsOrderFixture +} from '../../test/fixtures/expShow/base/customPlots' import { ExperimentStatus, EXPERIMENT_WORKSPACE_ID @@ -48,26 +50,7 @@ describe('collectCustomPlotsData', () => { } ) const data = collectCustomPlotsData( - [ - { - metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout', - type: CustomPlotType.METRIC_VS_PARAM - }, - { - metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:epochs', - type: CustomPlotType.METRIC_VS_PARAM - }, - { - metric: 'metrics:summary.json:loss', - type: CustomPlotType.CHECKPOINT - }, - { - metric: 'metrics:summary.json:accuracy', - type: CustomPlotType.CHECKPOINT - } - ], + customPlotsOrderFixture, { 'summary.json:accuracy': { id: 'custom-summary.json:accuracy', @@ -114,36 +97,36 @@ describe('collectCustomPlotsData', () => { label: '123', metrics: { 'summary.json': { - accuracy: 0.4668000042438507, + accuracy: 0.3724166750907898, loss: 2.0205044746398926 } }, name: 'exp-e7a67', - params: { 'params.yaml': { dropout: 0.15, epochs: 16 } } + params: { 'params.yaml': { dropout: 0.15, epochs: 2 } } }, { id: '12345', label: '123', metrics: { 'summary.json': { - accuracy: 0.3484833240509033, + accuracy: 0.4668000042438507, loss: 1.9293040037155151 } }, - name: 'exp-83425', - params: { 'params.yaml': { dropout: 0.25, epochs: 10 } } + name: 'test-branch', + params: { 'params.yaml': { dropout: 0.122, epochs: 2 } } }, { id: '12345', label: '123', metrics: { 'summary.json': { - accuracy: 0.6768440509033, - loss: 2.298503875732422 + accuracy: 0.5926499962806702, + loss: 1.775016188621521 } }, - name: 'exp-f13bca', - params: { 'params.yaml': { dropout: 0.32, epochs: 20 } } + name: 'exp-83425', + params: { 'params.yaml': { dropout: 0.124, epochs: 5 } } } ] ) diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index a87252c2ce..6ccadabcf1 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -71,13 +71,11 @@ export class PlotsModel extends ModelWithPersistence { private multiSourceVariations: MultiSourceVariations = {} private multiSourceEncoding: MultiSourceEncoding = {} - private checkpointPlots?: CheckpointPlot[] // TBD we need to move this type to a named type if // we plan to keep this private customCheckpointPlots?: { [metric: string]: CheckpointPlot } private customPlots?: CustomPlot[] private selectedMetrics?: string[] - private metricOrder: string[] constructor( dvcRoot: string, @@ -102,7 +100,6 @@ export class PlotsModel extends ModelWithPersistence { PersistenceKey.PLOT_SELECTED_METRICS, undefined ) - this.metricOrder = this.revive(PersistenceKey.PLOT_METRIC_ORDER, []) this.customPlotsOrder = this.revive(PersistenceKey.PLOTS_CUSTOM_ORDER, []) } @@ -126,7 +123,6 @@ export class PlotsModel extends ModelWithPersistence { collectTemplates(data), collectMultiSourceVariations(data, this.multiSourceVariations) ]) - this.recreateCustomPlots() this.comparisonData = { @@ -179,8 +175,9 @@ export class PlotsModel extends ModelWithPersistence { if (data) { this.customCheckpointPlots = collectCustomCheckpointPlotData(data) } + const experiments = this.experiments.getExperiments() - // TBD this if check is going to nned to be rethought since checkpoint data + // TBD this if check is going to need to be rethought since checkpoint data // is involved now if (experiments.length === 0) { this.customPlots = undefined @@ -198,12 +195,16 @@ export class PlotsModel extends ModelWithPersistence { return this.customPlotsOrder } - public setCustomPlotsOrder(plotsOrder: CustomPlotsOrderValue[]) { + public updateCustomPlotsOrder(plotsOrder: CustomPlotsOrderValue[]) { this.customPlotsOrder = plotsOrder - this.persist(PersistenceKey.PLOTS_CUSTOM_ORDER, this.customPlotsOrder) this.recreateCustomPlots() } + public setCustomPlotsOrder(plotsOrder: CustomPlotsOrderValue[]) { + this.updateCustomPlotsOrder(plotsOrder) + this.persist(PersistenceKey.PLOTS_CUSTOM_ORDER, this.customPlotsOrder) + } + public removeCustomPlots(plotIds: string[]) { const newCustomPlotsOrder = this.getCustomPlotsOrder().filter( plot => !plotIds.includes(getCustomPlotId(plot)) diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 7ca5737e70..0f87a32c6c 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -115,6 +115,37 @@ export class WebviewMessages { } } + public async getMetricOrParamPlot(): Promise< + CustomPlotsOrderValue | undefined + > { + const metricAndParam = await pickMetricAndParam( + this.experiments.getColumnTerminalNodes() + ) + + if (!metricAndParam) { + return + } + + const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { + if (isCustomPlotOrderCheckpointValue(value)) { + return + } + return ( + value.param === metricAndParam.param && + value.metric === metricAndParam.metric + ) + }) + + if (plotAlreadyExists) { + return Toast.showError('Custom plot already exists.') + } + + return { + ...metricAndParam, + type: CustomPlotType.METRIC_VS_PARAM + } + } + private setPlotSize( section: Section, nbItemsPerRow: number, @@ -169,37 +200,6 @@ export class WebviewMessages { ) } - private async getMetricOrParamPlot(): Promise< - CustomPlotsOrderValue | undefined - > { - const metricAndParam = await pickMetricAndParam( - this.experiments.getColumnTerminalNodes() - ) - - if (!metricAndParam) { - return - } - - const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { - if (isCustomPlotOrderCheckpointValue(value)) { - return - } - return ( - value.param === metricAndParam.param && - value.metric === metricAndParam.metric - ) - }) - - if (plotAlreadyExists) { - return Toast.showError('Custom plot already exists.') - } - - return { - ...metricAndParam, - type: CustomPlotType.METRIC_VS_PARAM - } - } - private async getCheckpointPlot(): Promise< CustomPlotsOrderValue | undefined > { diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index 8f13a160a7..b63b6c8d30 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -1,10 +1,31 @@ import { copyOriginalColors } from '../../../../experiments/model/status/colors' +import { CustomPlotsOrderValue } from '../../../../plots/model/custom' import { CustomPlotsData, CustomPlotType, PlotNumberOfItemsPerRow } from '../../../../plots/webview/contract' +export const customPlotsOrderFixture: CustomPlotsOrderValue[] = [ + { + metric: 'metrics:summary.json:loss', + param: 'params:params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + }, + { + metric: 'metrics:summary.json:accuracy', + param: 'params:params.yaml:epochs', + type: CustomPlotType.METRIC_VS_PARAM + }, + { + metric: 'metrics:summary.json:loss', + type: CustomPlotType.CHECKPOINT + }, + { + metric: 'metrics:summary.json:accuracy', + type: CustomPlotType.CHECKPOINT + } +] const colors = copyOriginalColors() const data: CustomPlotsData = { @@ -27,14 +48,14 @@ const data: CustomPlotsData = { param: 0.15 }, { - expName: 'exp-83425', + expName: 'test-branch', metric: 1.9293040037155151, - param: 0.25 + param: 0.122 }, { - expName: 'exp-f13bca', - metric: 2.298503875732422, - param: 0.32 + expName: 'exp-83425', + metric: 1.775016188621521, + param: 0.124 } ], yTitle: 'summary.json:loss' @@ -47,18 +68,18 @@ const data: CustomPlotsData = { values: [ { expName: 'exp-e7a67', - metric: 0.4668000042438507, - param: 16 + metric: 0.3724166750907898, + param: 2 }, { - expName: 'exp-83425', - metric: 0.3484833240509033, - param: 10 + expName: 'test-branch', + metric: 0.4668000042438507, + param: 2 }, { - expName: 'exp-f13bca', - metric: 0.6768440509033, - param: 20 + expName: 'exp-83425', + metric: 0.5926499962806702, + param: 5 } ], yTitle: 'summary.json:accuracy' diff --git a/extension/src/test/suite/experiments/model/tree.test.ts b/extension/src/test/suite/experiments/model/tree.test.ts index c4ede12f00..be4bd21115 100644 --- a/extension/src/test/suite/experiments/model/tree.test.ts +++ b/extension/src/test/suite/experiments/model/tree.test.ts @@ -70,12 +70,17 @@ suite('Experiments Tree Test Suite', () => { }) it('should be able to toggle whether an experiment is shown in the plots webview with dvc.views.experiments.toggleStatus', async () => { - const { plots, messageSpy } = await buildPlots(disposable) + const { plots, messageSpy } = await buildPlots( + disposable, + undefined, + undefined + ) const expectedDomain = [...domain] const expectedRange = [...range] const webview = await plots.showWebview() + await webview.isReady() while (expectedDomain.length > 0) { @@ -84,10 +89,10 @@ suite('Experiments Tree Test Suite', () => { expectedRange ) - const { checkpoint } = getFirstArgOfLastCall(messageSpy) + const { custom } = getFirstArgOfLastCall(messageSpy) expect( - { checkpoint }, + { custom }, 'a message is sent with colors for the currently selected experiments' ).to.deep.equal(expectedData) messageSpy.resetHistory() @@ -106,12 +111,8 @@ suite('Experiments Tree Test Suite', () => { expect(unSelected).to.equal(UNSELECTED) } - expect( - messageSpy, - 'when there are no experiments selected we send null (show empty state)' - ).to.be.calledWithMatch({ - checkpoint: null - }) + // TBD rewrite this test 'when there are no experiments selected we send null (show empty state)' + // to expect custom data but with no trends plots messageSpy.resetHistory() expectedDomain.push(domain[0]) diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index b6725b2fef..b10544871b 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -7,7 +7,9 @@ import { restore, spy, stub } from 'sinon' import { buildPlots } from '../plots/util' import { Disposable } from '../../../extension' import expShowFixtureWithoutErrors from '../../fixtures/expShow/base/noErrors' -import customPlotsFixture from '../../fixtures/expShow/base/customPlots' +import customPlotsFixture, { + customPlotsOrderFixture +} from '../../fixtures/expShow/base/customPlots' import plotsDiffFixture from '../../fixtures/plotsDiff/output' import multiSourcePlotsDiffFixture from '../../fixtures/plotsDiff/multiSource' import templatePlotsFixture from '../../fixtures/plotsDiff/template' @@ -456,11 +458,13 @@ suite('Plots Test Suite', () => { expect(mockSetCustomPlotsOrder).to.be.calledWithExactly([ { metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:epochs' + param: 'params:params.yaml:epochs', + type: CustomPlotType.METRIC_VS_PARAM }, { metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout' + param: 'params:params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM } ]) expect(messageSpy).to.be.calledOnce @@ -762,25 +766,41 @@ suite('Plots Test Suite', () => { }).timeout(WEBVIEW_TEST_TIMEOUT) it('should handle a add custom plot message from the webview', async () => { - const { plots, plotsModel } = await buildPlots( + const { plots, plotsModel, webviewMessages } = await buildPlots( disposable, plotsDiffFixture ) const webview = await plots.showWebview() - const mockGetMetricAndParam = stub( + const mockCustomPlotOrderValue = { + metric: 'metrics:summary.json:accuracy', + param: 'params:params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + } + + const mockPickCustomPlotType = stub( customPlotQuickPickUtil, - 'pickMetricAndParam' + 'pickCustomPlotType' ) - const quickPickEvent = new Promise(resolve => + const mockGetMetricAndParam = stub( + webviewMessages, + 'getMetricOrParamPlot' + ) + + const firstQuickPickEvent = new Promise(resolve => + mockPickCustomPlotType.onFirstCall().callsFake(() => { + resolve(undefined) + + return Promise.resolve(CustomPlotType.METRIC_VS_PARAM) + }) + ) + + const secondQuickPickEvent = new Promise(resolve => mockGetMetricAndParam.callsFake(() => { resolve(undefined) - return Promise.resolve({ - metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout' - }) + return Promise.resolve(mockCustomPlotOrderValue) }) ) @@ -792,13 +812,12 @@ suite('Plots Test Suite', () => { mockMessageReceived.fire({ type: MessageFromWebviewType.ADD_CUSTOM_PLOT }) - await quickPickEvent + await firstQuickPickEvent + await secondQuickPickEvent expect(mockSetCustomPlotsOrder).to.be.calledWith([ - { - metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout' - } + ...customPlotsOrderFixture, + mockCustomPlotOrderValue ]) expect(mockSendTelemetryEvent).to.be.calledWith( EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED, diff --git a/extension/src/test/suite/plots/util.ts b/extension/src/test/suite/plots/util.ts index 980ebe983f..cb11eabe11 100644 --- a/extension/src/test/suite/plots/util.ts +++ b/extension/src/test/suite/plots/util.ts @@ -2,7 +2,9 @@ import { Disposer } from '@hediet/std/disposable' import { stub } from 'sinon' import * as FileSystem from '../../../fileSystem' import expShowFixtureWithoutErrors from '../../fixtures/expShow/base/noErrors' -import customPlotsFixture from '../../fixtures/expShow/base/customPlots' +import customPlotsFixture, { + customPlotsOrderFixture +} from '../../fixtures/expShow/base/customPlots' import { Plots } from '../../../plots' import { buildMockMemento, dvcDemoPath } from '../../util' import { WorkspacePlots } from '../../../plots/workspace' @@ -90,6 +92,7 @@ export const buildPlots = async ( // eslint-disable-next-line @typescript-eslint/no-explicit-any const plotsModel: PlotsModel = (plots as any).plots + plotsModel.updateCustomPlotsOrder(customPlotsOrderFixture) // eslint-disable-next-line @typescript-eslint/no-explicit-any const pathsModel: PathsModel = (plots as any).paths @@ -134,7 +137,7 @@ export const getExpectedCustomPlotsData = ( ) => { const { plots, nbItemsPerRow, height } = customPlotsFixture return { - checkpoint: { + custom: { colors: { domain, range @@ -142,13 +145,11 @@ export const getExpectedCustomPlotsData = ( height, nbItemsPerRow, plots: plots.map(plot => ({ - id: plot.id, - type: plot.type, + ...plot, values: plot.type === CustomPlotType.CHECKPOINT ? plot.values.filter(value => domain.includes(value.group)) - : plot.values, - yTitle: plot.yTitle + : plot.values })) } } From e50544263cd3705899c31e6952763b0de44b2189 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Wed, 8 Mar 2023 11:48:09 -0600 Subject: [PATCH 06/40] Fix reordering bug --- extension/src/plots/model/index.ts | 27 ++++++++++++------- .../components/customPlots/CustomPlots.tsx | 3 +-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 6ccadabcf1..2947e07da5 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -30,7 +30,8 @@ import { PlotNumberOfItemsPerRow, DEFAULT_HEIGHT, CustomPlotsData, - CustomPlot + CustomPlot, + ColorScale } from '../webview/contract' import { ExperimentsOutput, @@ -156,7 +157,6 @@ export class PlotsModel extends ModelWithPersistence { return } - // TBD colors is undefined if there are no selected exps const colors = getColorScale( this.experiments .getSelectedExperiments() @@ -167,7 +167,7 @@ export class PlotsModel extends ModelWithPersistence { colors, height: this.getHeight(Section.CUSTOM_PLOTS), nbItemsPerRow: this.getNbItemsPerRow(Section.CUSTOM_PLOTS), - plots: this.getCustomPlotData(this.customPlots, colors?.domain) + plots: this.getCustomPlotData(this.customPlots, colors) } } @@ -479,19 +479,28 @@ export class PlotsModel extends ModelWithPersistence { private getCustomPlotData( plots: CustomPlot[], - selectedExperiments: string[] | undefined + colors: ColorScale | undefined ): CustomPlotData[] { - if (!selectedExperiments) { - return plots.filter(plot => !isCheckpointPlot(plot)) as CustomPlotData[] + if (!colors) { + return plots + .filter(plot => !isCheckpointPlot(plot)) + .map( + plot => + ({ + ...plot, + yTitle: truncateVerticalTitle( + plot.metric, + this.getNbItemsPerRow(Section.CUSTOM_PLOTS) + ) as string + } as CustomPlotData) + ) } return plots.map( plot => ({ ...plot, values: isCheckpointPlot(plot) - ? plot.values.filter(value => - selectedExperiments.includes(value.group) - ) + ? plot.values.filter(value => colors.domain.includes(value.group)) : plot.values, yTitle: truncateVerticalTitle( plot.metric, diff --git a/webview/src/plots/components/customPlots/CustomPlots.tsx b/webview/src/plots/components/customPlots/CustomPlots.tsx index 6bb54ef9a4..1c07e4a3f0 100644 --- a/webview/src/plots/components/customPlots/CustomPlots.tsx +++ b/webview/src/plots/components/customPlots/CustomPlots.tsx @@ -2,7 +2,6 @@ import React, { DragEvent, useEffect, useState } from 'react' import { useSelector } from 'react-redux' import cx from 'classnames' import { MessageFromWebviewType } from 'dvc/src/webview/contract' -import { performSimpleOrderedUpdate } from 'dvc/src/util/array' import { CustomPlot } from './CustomPlot' import styles from '../styles.module.scss' import { EmptyState } from '../../../shared/components/emptyState/EmptyState' @@ -32,7 +31,7 @@ export const CustomPlots: React.FC = ({ plotsIds }) => { ) useEffect(() => { - setOrder(pastOrder => performSimpleOrderedUpdate(pastOrder, plotsIds)) + setOrder(plotsIds) }, [plotsIds]) const setPlotsIdsOrder = (order: string[]): void => { From 8ab22c4c6ad8694bc40a77b94e3525f052b2dd3e Mon Sep 17 00:00:00 2001 From: julieg18 Date: Wed, 8 Mar 2023 18:41:57 -0600 Subject: [PATCH 07/40] Resolve comments --- extension/src/plots/model/collect.ts | 17 ++--- extension/src/plots/model/custom.ts | 15 +---- extension/src/plots/model/index.ts | 9 ++- extension/src/plots/webview/messages.ts | 67 +++++++++---------- .../test/fixtures/expShow/base/customPlots.ts | 2 - .../test/suite/experiments/model/tree.test.ts | 33 ++++++--- extension/src/test/suite/plots/index.test.ts | 6 +- .../components/customPlots/CustomPlot.tsx | 3 +- .../src/plots/components/customPlots/util.ts | 2 +- webview/src/stories/Plots.stories.tsx | 2 +- 10 files changed, 75 insertions(+), 81 deletions(-) diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 1fb3e4027c..137b868ad0 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -2,10 +2,8 @@ import omit from 'lodash.omit' import get from 'lodash.get' import { TopLevelSpec } from 'vega-lite' import { VisualizationSpec } from 'react-vega' -import { - CustomPlotsOrderValue, - isCustomPlotOrderCheckpointValue -} from './custom' +import { CustomCheckpointPlots } from '.' +import { CustomPlotsOrderValue } from './custom' import { getRevisionFirstThreeColumns } from './util' import { ColorScale, @@ -223,10 +221,9 @@ export const getCustomPlotId = (plot: CustomPlotsOrderValue) => ? `custom-${plot.metric}` : `custom-${plot.metric}-${plot.param}` -// TBD untested... export const collectCustomCheckpointPlotData = ( data: ExperimentsOutput -): { [metric: string]: CheckpointPlot } => { +): CustomCheckpointPlots => { const acc = { iterations: {}, plots: new Map() @@ -242,7 +239,7 @@ export const collectCustomCheckpointPlotData = ( } } - const plotsData: { [metric: string]: CheckpointPlot } = {} + const plotsData: CustomCheckpointPlots = {} if (acc.plots.size === 0) { return plotsData } @@ -302,16 +299,14 @@ const collectMetricVsParamPlotData = ( return plotData } -// TBD it will probably be easier and/or faster to get the data from -// experiments vs the output... export const collectCustomPlotsData = ( plotsOrderValues: CustomPlotsOrderValue[], - checkpointPlots: { [metric: string]: CheckpointPlot }, + checkpointPlots: CustomCheckpointPlots, experiments: Experiment[] ): CustomPlot[] => { return plotsOrderValues .map((plotOrderValue): CustomPlot => { - if (isCustomPlotOrderCheckpointValue(plotOrderValue)) { + if (plotOrderValue.type === CustomPlotType.CHECKPOINT) { const { metric } = plotOrderValue return checkpointPlots[metric.slice(ColumnType.METRICS.length + 1)] } diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts index 1da1f06f3b..4d8e445e27 100644 --- a/extension/src/plots/model/custom.ts +++ b/extension/src/plots/model/custom.ts @@ -1,23 +1,14 @@ import { CustomPlotType } from '../webview/contract' -// these names are way too lengthy -export type CustomPlotOrderCheckpointValue = { +type CheckpointValue = { type: CustomPlotType.CHECKPOINT metric: string } -export type CustomPlotOrderMetricVsParamValue = { +type MetricVsParamValue = { type: CustomPlotType.METRIC_VS_PARAM metric: string param: string } -export type CustomPlotsOrderValue = - | CustomPlotOrderCheckpointValue - | CustomPlotOrderMetricVsParamValue - -export const isCustomPlotOrderCheckpointValue = ( - plot: CustomPlotsOrderValue -): plot is CustomPlotOrderCheckpointValue => { - return plot.type === CustomPlotType.CHECKPOINT -} +export type CustomPlotsOrderValue = CheckpointValue | MetricVsParamValue diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 2947e07da5..c7278400da 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -53,6 +53,8 @@ import { } from '../multiSource/collect' import { isDvcError } from '../../cli/dvc/reader' +export type CustomCheckpointPlots = { [metric: string]: CheckpointPlot } + export class PlotsModel extends ModelWithPersistence { private readonly experiments: Experiments @@ -72,9 +74,7 @@ export class PlotsModel extends ModelWithPersistence { private multiSourceVariations: MultiSourceVariations = {} private multiSourceEncoding: MultiSourceEncoding = {} - // TBD we need to move this type to a named type if - // we plan to keep this - private customCheckpointPlots?: { [metric: string]: CheckpointPlot } + private customCheckpointPlots?: CustomCheckpointPlots private customPlots?: CustomPlot[] private selectedMetrics?: string[] @@ -177,8 +177,7 @@ export class PlotsModel extends ModelWithPersistence { } const experiments = this.experiments.getExperiments() - // TBD this if check is going to need to be rethought since checkpoint data - // is involved now + if (experiments.length === 0) { this.customPlots = undefined return diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 0f87a32c6c..15f057617c 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -33,10 +33,7 @@ import { ColumnType } from '../../experiments/webview/contract' import { FILE_SEPARATOR } from '../../experiments/columns/paths' import { reorderObjectList } from '../../util/array' import { isCheckpointPlot } from '../model/collect' -import { - CustomPlotsOrderValue, - isCustomPlotOrderCheckpointValue -} from '../model/custom' +import { CustomPlotsOrderValue } from '../model/custom' export class WebviewMessages { private readonly paths: PathsModel @@ -115,7 +112,7 @@ export class WebviewMessages { } } - public async getMetricOrParamPlot(): Promise< + public async addMetricVsParamPlot(): Promise< CustomPlotsOrderValue | undefined > { const metricAndParam = await pickMetricAndParam( @@ -127,7 +124,7 @@ export class WebviewMessages { } const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { - if (isCustomPlotOrderCheckpointValue(value)) { + if (value.type === CustomPlotType.CHECKPOINT) { return } return ( @@ -140,10 +137,12 @@ export class WebviewMessages { return Toast.showError('Custom plot already exists.') } - return { + const plot = { ...metricAndParam, type: CustomPlotType.METRIC_VS_PARAM } + this.plots.addCustomPlot(plot) + this.sendCustomPlotsAndEvent(EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED) } private setPlotSize( @@ -200,7 +199,7 @@ export class WebviewMessages { ) } - private async getCheckpointPlot(): Promise< + private async addCheckpointPlot(): Promise< CustomPlotsOrderValue | undefined > { const metric = await pickMetric(this.experiments.getColumnTerminalNodes()) @@ -210,7 +209,7 @@ export class WebviewMessages { } const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { - if (isCustomPlotOrderCheckpointValue(value)) { + if (value.type === CustomPlotType.CHECKPOINT) { return value.metric === metric } }) @@ -219,35 +218,31 @@ export class WebviewMessages { return Toast.showError('Custom plot already exists.') } - return { + const plot: CustomPlotsOrderValue = { metric, type: CustomPlotType.CHECKPOINT } + this.plots.addCustomPlot(plot) + this.sendCustomPlotsAndEvent(EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED) } private async addCustomPlot() { - const plotType = await pickCustomPlotType() - - if (!plotType) { + if (!this.experiments.hasCheckpoints()) { + void this.addMetricVsParamPlot() return } - const plot = await (plotType === CustomPlotType.CHECKPOINT - ? this.getCheckpointPlot() - : this.getMetricOrParamPlot()) + const plotType = await pickCustomPlotType() - if (!plot) { + if (!plotType) { return } - this.plots.addCustomPlot(plot) - - this.sendCustomPlots() - sendTelemetryEvent( - EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED, - undefined, - undefined - ) + if (plotType === CustomPlotType.CHECKPOINT) { + void this.addCheckpointPlot() + } else { + void this.addMetricVsParamPlot() + } } private async removeCustomPlots() { @@ -264,12 +259,7 @@ export class WebviewMessages { } this.plots.removeCustomPlots(selectedPlotsIds) - this.sendCustomPlots() - sendTelemetryEvent( - EventName.VIEWS_PLOTS_CUSTOM_PLOT_REMOVED, - undefined, - undefined - ) + this.sendCustomPlotsAndEvent(EventName.VIEWS_PLOTS_CUSTOM_PLOT_REMOVED) } private setCustomPlotsOrder(plotIds: string[]) { @@ -297,12 +287,17 @@ export class WebviewMessages { } ) this.plots.setCustomPlotsOrder(newOrder) + this.sendCustomPlotsAndEvent(EventName.VIEWS_REORDER_PLOTS_CUSTOM) + } + + private sendCustomPlotsAndEvent( + event: + | typeof EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED + | typeof EventName.VIEWS_PLOTS_CUSTOM_PLOT_REMOVED + | typeof EventName.VIEWS_REORDER_PLOTS_CUSTOM + ) { this.sendCustomPlots() - sendTelemetryEvent( - EventName.VIEWS_REORDER_PLOTS_CUSTOM, - undefined, - undefined - ) + sendTelemetryEvent(event, undefined, undefined) } private selectPlotsFromWebview() { diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index b63b6c8d30..9c8ca6a672 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -36,8 +36,6 @@ const data: CustomPlotsData = { plots: [ { id: 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - // TBD I don't think we actually need metric/param here - // since I think only title is used in in the front end metric: 'summary.json:loss', param: 'params.yaml:dropout', type: CustomPlotType.METRIC_VS_PARAM, diff --git a/extension/src/test/suite/experiments/model/tree.test.ts b/extension/src/test/suite/experiments/model/tree.test.ts index be4bd21115..3969d6d831 100644 --- a/extension/src/test/suite/experiments/model/tree.test.ts +++ b/extension/src/test/suite/experiments/model/tree.test.ts @@ -26,6 +26,7 @@ import { import { buildPlots, getExpectedCustomPlotsData } from '../../plots/util' import customPlotsFixture from '../../../fixtures/expShow/base/customPlots' import expShowFixture from '../../../fixtures/expShow/base/output' +import plotsRevisionsFixture from '../../../fixtures/plotsDiff/revisions' import { ExperimentsTree } from '../../../../experiments/model/tree' import { buildExperiments, @@ -45,7 +46,11 @@ import { WorkspaceExperiments } from '../../../../experiments/workspace' import { ExperimentItem } from '../../../../experiments/model/collect' import { EXPERIMENT_WORKSPACE_ID } from '../../../../cli/dvc/contract' import { DvcReader } from '../../../../cli/dvc/reader' -import { ColorScale } from '../../../../plots/webview/contract' +import { + ColorScale, + CustomPlotType, + DEFAULT_SECTION_COLLAPSED +} from '../../../../plots/webview/contract' suite('Experiments Tree Test Suite', () => { const disposable = getTimeSafeDisposer() @@ -70,11 +75,7 @@ suite('Experiments Tree Test Suite', () => { }) it('should be able to toggle whether an experiment is shown in the plots webview with dvc.views.experiments.toggleStatus', async () => { - const { plots, messageSpy } = await buildPlots( - disposable, - undefined, - undefined - ) + const { plots, messageSpy } = await buildPlots(disposable) const expectedDomain = [...domain] const expectedRange = [...range] @@ -111,8 +112,24 @@ suite('Experiments Tree Test Suite', () => { expect(unSelected).to.equal(UNSELECTED) } - // TBD rewrite this test 'when there are no experiments selected we send null (show empty state)' - // to expect custom data but with no trends plots + expect( + messageSpy, + 'when there are no experiments selected we dont send trend type plots' + ).to.be.calledWithMatch({ + comparison: null, + custom: { + ...customPlotsFixture, + colors: undefined, + plots: customPlotsFixture.plots.filter( + plot => plot.type !== CustomPlotType.CHECKPOINT + ) + }, + hasPlots: false, + hasUnselectedPlots: false, + sectionCollapsed: DEFAULT_SECTION_COLLAPSED, + selectedRevisions: plotsRevisionsFixture.slice(0, 2), + template: null + }) messageSpy.resetHistory() expectedDomain.push(domain[0]) diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index b10544871b..30cb88bf12 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -766,7 +766,7 @@ suite('Plots Test Suite', () => { }).timeout(WEBVIEW_TEST_TIMEOUT) it('should handle a add custom plot message from the webview', async () => { - const { plots, plotsModel, webviewMessages } = await buildPlots( + const { plots, plotsModel } = await buildPlots( disposable, plotsDiffFixture ) @@ -785,8 +785,8 @@ suite('Plots Test Suite', () => { ) const mockGetMetricAndParam = stub( - webviewMessages, - 'getMetricOrParamPlot' + customPlotQuickPickUtil, + 'pickMetricAndParam' ) const firstQuickPickEvent = new Promise(resolve => diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index b31e6f1811..26c6aa923d 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -30,8 +30,7 @@ const createCustomPlotSpec = ( if (!plot) { return {} } - // TBD were forced to use this type of "if or" statement mutliple times throughout the custom code - // There's probably a better way to do this + if (isCheckpointPlot(plot)) { return colors ? createCheckpointSpec(plot.yTitle, colors) : {} } diff --git a/webview/src/plots/components/customPlots/util.ts b/webview/src/plots/components/customPlots/util.ts index 92deef218b..b945a15ecc 100644 --- a/webview/src/plots/components/customPlots/util.ts +++ b/webview/src/plots/components/customPlots/util.ts @@ -91,7 +91,7 @@ export const createCheckpointSpec = ( ], width: 'container' } as VisualizationSpec) -// TBD rename to title? + export const createMetricVsParamSpec = (metric: string, param: string) => ({ $schema: 'https://vega.github.io/schema/vega-lite/v5.json', diff --git a/webview/src/stories/Plots.stories.tsx b/webview/src/stories/Plots.stories.tsx index be2c2c2c48..b62c35b084 100644 --- a/webview/src/stories/Plots.stories.tsx +++ b/webview/src/stories/Plots.stories.tsx @@ -26,7 +26,7 @@ import '../shared/style.scss' import '../plots/components/styles.module.scss' import { feedStore } from '../plots/components/App' import { plotsReducers } from '../plots/store' -// TBD review storybooks, making sure they all make sense + const manyCustomPlots = (length: number, size = PlotNumberOfItemsPerRow.TWO) => Array.from({ length }, () => customPlotsFixture.plots[2]).map((plot, i) => { const id = plot.id + i.toString() From 3354b3e6129496447c1ca3d42a319d605ce0adc7 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Thu, 9 Mar 2023 09:13:01 -0600 Subject: [PATCH 08/40] Fix e2e tests --- extension/src/test/e2e/extension.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/src/test/e2e/extension.test.ts b/extension/src/test/e2e/extension.test.ts index 17df6ca615..6572b762d9 100644 --- a/extension/src/test/e2e/extension.test.ts +++ b/extension/src/test/e2e/extension.test.ts @@ -125,7 +125,7 @@ describe('Plots Webview', function () { await browser.waitUntil( async () => { - return (await webview.vegaVisualization$$.length) === 10 + return (await webview.vegaVisualization$$.length) === 5 }, { timeout: 30000 } ) From bd433cbaea5fc5f51ad459bb7d30aa2254ae52e8 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Thu, 9 Mar 2023 15:47:28 -0600 Subject: [PATCH 09/40] Add accidentally deleted test --- extension/src/plots/model/collect.test.ts | 53 +++++++++++- .../test/fixtures/expShow/base/customPlots.ts | 80 +++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/extension/src/plots/model/collect.test.ts b/extension/src/plots/model/collect.test.ts index 6e99de325f..96db2511de 100644 --- a/extension/src/plots/model/collect.test.ts +++ b/extension/src/plots/model/collect.test.ts @@ -1,14 +1,17 @@ import { join } from 'path' +import omit from 'lodash.omit' import isEmpty from 'lodash.isempty' import { collectData, collectTemplates, collectOverrideRevisionDetails, - collectCustomPlotsData + collectCustomPlotsData, + collectCustomCheckpointPlotData } from './collect' import plotsDiffFixture from '../../test/fixtures/plotsDiff/output' import customPlotsFixture, { - customPlotsOrderFixture + customPlotsOrderFixture, + checkpointPlotsFixture } from '../../test/fixtures/expShow/base/customPlots' import { ExperimentStatus, @@ -23,6 +26,8 @@ import { TemplatePlot } from '../webview/contract' import { getCLICommitId } from '../../test/fixtures/plotsDiff/util' +import expShowFixture from '../../test/fixtures/expShow/base/output' +import modifiedFixture from '../../test/fixtures/expShow/modified/output' import { SelectedExperimentWithColor } from '../../experiments/model' import { Experiment } from '../../experiments/webview/contract' @@ -197,6 +202,50 @@ describe('collectData', () => { }) }) +describe('collectCustomCheckpointPlotsData', () => { + it('should return the expected data from the test fixture', () => { + const data = collectCustomCheckpointPlotData(expShowFixture) + + expect(data).toStrictEqual(checkpointPlotsFixture) + }) + + it('should provide a continuous series for a modified experiment', () => { + const data = collectCustomCheckpointPlotData(modifiedFixture) + + for (const { values } of Object.values(data)) { + const initialExperiment = values.filter( + point => point.group === 'exp-908bd' + ) + const modifiedExperiment = values.find( + point => point.group === 'exp-01b3a' + ) + + const lastIterationInitial = initialExperiment?.slice(-1)[0] + const firstIterationModified = modifiedExperiment + + expect(lastIterationInitial).not.toStrictEqual(firstIterationModified) + expect(omit(lastIterationInitial, 'group')).toStrictEqual( + omit(firstIterationModified, 'group') + ) + + const baseExperiment = values.filter(point => point.group === 'exp-920fc') + const restartedExperiment = values.find( + point => point.group === 'exp-9bc1b' + ) + + const iterationRestartedFrom = baseExperiment?.slice(5)[0] + const firstIterationAfterRestart = restartedExperiment + + expect(iterationRestartedFrom).not.toStrictEqual( + firstIterationAfterRestart + ) + expect(omit(iterationRestartedFrom, 'group')).toStrictEqual( + omit(firstIterationAfterRestart, 'group') + ) + } + }) +}) + describe('collectTemplates', () => { it('should return the expected output from the test fixture', () => { const { content } = logsLossPlot diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index d3b3d1b54a..d04b98434e 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -26,6 +26,86 @@ export const customPlotsOrderFixture: CustomPlotsOrderValue[] = [ type: CustomPlotType.CHECKPOINT } ] + +export const checkpointPlotsFixture = { + 'summary.json:loss': { + id: 'custom-summary.json:loss', + metric: 'summary.json:loss', + type: 'checkpoint', + values: [ + { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, + { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, + { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, + { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, + { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, + { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, + { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, + { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, + { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, + { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, + { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, + { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } + ] + }, + 'summary.json:accuracy': { + id: 'custom-summary.json:accuracy', + metric: 'summary.json:accuracy', + type: 'checkpoint', + values: [ + { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, + { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, + { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, + { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, + { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, + { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, + { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, + { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, + { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, + { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, + { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, + { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } + ] + }, + 'summary.json:val_loss': { + id: 'custom-summary.json:val_loss', + metric: 'summary.json:val_loss', + type: 'checkpoint', + values: [ + { group: 'exp-83425', iteration: 1, y: 1.9391471147537231 }, + { group: 'exp-83425', iteration: 2, y: 1.8825950622558594 }, + { group: 'exp-83425', iteration: 3, y: 1.827923059463501 }, + { group: 'exp-83425', iteration: 4, y: 1.7749212980270386 }, + { group: 'exp-83425', iteration: 5, y: 1.7233840227127075 }, + { group: 'exp-83425', iteration: 6, y: 1.7233840227127075 }, + { group: 'test-branch', iteration: 1, y: 1.9363881349563599 }, + { group: 'test-branch', iteration: 2, y: 1.8770883083343506 }, + { group: 'test-branch', iteration: 3, y: 1.8770883083343506 }, + { group: 'exp-e7a67', iteration: 1, y: 1.9979370832443237 }, + { group: 'exp-e7a67', iteration: 2, y: 1.9979370832443237 }, + { group: 'exp-e7a67', iteration: 3, y: 1.9979370832443237 } + ] + }, + 'summary.json:val_accuracy': { + id: 'custom-summary.json:val_accuracy', + metric: 'summary.json:val_accuracy', + type: 'checkpoint', + values: [ + { group: 'exp-83425', iteration: 1, y: 0.49399998784065247 }, + { group: 'exp-83425', iteration: 2, y: 0.5550000071525574 }, + { group: 'exp-83425', iteration: 3, y: 0.6035000085830688 }, + { group: 'exp-83425', iteration: 4, y: 0.6414999961853027 }, + { group: 'exp-83425', iteration: 5, y: 0.6704000234603882 }, + { group: 'exp-83425', iteration: 6, y: 0.6704000234603882 }, + { group: 'test-branch', iteration: 1, y: 0.4970000088214874 }, + { group: 'test-branch', iteration: 2, y: 0.5608000159263611 }, + { group: 'test-branch', iteration: 3, y: 0.5608000159263611 }, + { group: 'exp-e7a67', iteration: 1, y: 0.4277999997138977 }, + { group: 'exp-e7a67', iteration: 2, y: 0.4277999997138977 }, + { group: 'exp-e7a67', iteration: 3, y: 0.4277999997138977 } + ] + } +} + const colors = copyOriginalColors() const data: CustomPlotsData = { From b4d64b3badf0e282347796fa675828f20833049e Mon Sep 17 00:00:00 2001 From: julieg18 Date: Thu, 9 Mar 2023 18:32:53 -0600 Subject: [PATCH 10/40] Rename trends plot --- extension/src/plots/model/quickPick.test.ts | 4 ++-- extension/src/plots/model/quickPick.ts | 4 ++-- extension/src/test/suite/experiments/model/tree.test.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index d118313ec7..68948043f0 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -67,7 +67,7 @@ describe('pickCustomPlots', () => { expect(mockedQuickPickManyValues).toHaveBeenCalledWith( [ { - description: 'Trend Plot', + description: 'Checkpoint Trend Plot', detail: 'metrics:summary.json:loss', label: 'loss', value: 'custom-metrics:summary.json:loss' @@ -113,7 +113,7 @@ describe('pickCustomPlotType', () => { { description: 'A linear plot that shows how a chosen metric changes over selected experiments.', - label: 'Trend', + label: 'Checkpoint Trend', value: CustomPlotType.CHECKPOINT } ], diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index 57ccfd0cbb..95ff3c64c6 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -33,7 +33,7 @@ const getMetricVsParamPlotItem = (metric: string, param: string) => { const getCheckpointPlotItem = (metric: string) => { const splitMetric = splitColumnPath(metric) return { - description: 'Trend Plot', + description: 'Checkpoint Trend Plot', detail: metric, label: splitMetric[splitMetric.length - 1], value: getCustomPlotId({ @@ -73,7 +73,7 @@ export const pickCustomPlotType = (): Thenable => { { description: 'A linear plot that shows how a chosen metric changes over selected experiments.', - label: 'Trend', + label: 'Checkpoint Trend', value: CustomPlotType.CHECKPOINT } ], diff --git a/extension/src/test/suite/experiments/model/tree.test.ts b/extension/src/test/suite/experiments/model/tree.test.ts index 3969d6d831..d19e6e4346 100644 --- a/extension/src/test/suite/experiments/model/tree.test.ts +++ b/extension/src/test/suite/experiments/model/tree.test.ts @@ -114,7 +114,7 @@ suite('Experiments Tree Test Suite', () => { expect( messageSpy, - 'when there are no experiments selected we dont send trend type plots' + 'when there are no experiments selected we dont send checkpoint type plots' ).to.be.calledWithMatch({ comparison: null, custom: { From 70526338d18834971c598fbb5c19f71e7693a121 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Thu, 9 Mar 2023 19:10:06 -0600 Subject: [PATCH 11/40] Refactor --- extension/src/plots/model/collect.ts | 11 +++-------- extension/src/plots/model/custom.ts | 9 ++++++++- extension/src/plots/model/index.ts | 5 ++--- extension/src/plots/model/quickPick.ts | 4 ++-- extension/src/plots/webview/messages.ts | 9 ++++----- extension/src/test/suite/plots/util.ts | 9 ++++----- .../src/plots/components/customPlots/CustomPlot.tsx | 7 +------ 7 files changed, 24 insertions(+), 30 deletions(-) diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 137b868ad0..6cbf399732 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -3,12 +3,11 @@ import get from 'lodash.get' import { TopLevelSpec } from 'vega-lite' import { VisualizationSpec } from 'react-vega' import { CustomCheckpointPlots } from '.' -import { CustomPlotsOrderValue } from './custom' +import { CustomPlotsOrderValue, isCheckpointValue } from './custom' import { getRevisionFirstThreeColumns } from './util' import { ColorScale, CheckpointPlotValues, - CheckpointPlot, isImagePlot, ImagePlot, TemplatePlot, @@ -217,7 +216,7 @@ const collectFromExperimentsObject = ( } export const getCustomPlotId = (plot: CustomPlotsOrderValue) => - plot.type === CustomPlotType.CHECKPOINT + isCheckpointValue(plot) ? `custom-${plot.metric}` : `custom-${plot.metric}-${plot.param}` @@ -260,10 +259,6 @@ export const collectCustomCheckpointPlotData = ( return plotsData } -export const isCheckpointPlot = (plot: CustomPlot): plot is CheckpointPlot => { - return plot.type === CustomPlotType.CHECKPOINT -} - const collectMetricVsParamPlotData = ( metric: string, param: string, @@ -306,7 +301,7 @@ export const collectCustomPlotsData = ( ): CustomPlot[] => { return plotsOrderValues .map((plotOrderValue): CustomPlot => { - if (plotOrderValue.type === CustomPlotType.CHECKPOINT) { + if (isCheckpointValue(plotOrderValue)) { const { metric } = plotOrderValue return checkpointPlots[metric.slice(ColumnType.METRICS.length + 1)] } diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts index 4d8e445e27..80a2a411b5 100644 --- a/extension/src/plots/model/custom.ts +++ b/extension/src/plots/model/custom.ts @@ -1,4 +1,4 @@ -import { CustomPlotType } from '../webview/contract' +import { CheckpointPlot, CustomPlot, CustomPlotType } from '../webview/contract' type CheckpointValue = { type: CustomPlotType.CHECKPOINT @@ -12,3 +12,10 @@ type MetricVsParamValue = { } export type CustomPlotsOrderValue = CheckpointValue | MetricVsParamValue + +export const isCheckpointValue = ( + value: CustomPlotsOrderValue +): value is CheckpointValue => value.type === CustomPlotType.CHECKPOINT + +export const isCheckpointPlot = (plot: CustomPlot): plot is CheckpointPlot => + plot.type === CustomPlotType.CHECKPOINT diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 4191b8f982..657715e580 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -12,11 +12,10 @@ import { collectOverrideRevisionDetails, collectCustomPlotsData, getCustomPlotId, - collectCustomCheckpointPlotData, - isCheckpointPlot + collectCustomCheckpointPlotData } from './collect' import { getRevisionFirstThreeColumns } from './util' -import { CustomPlotsOrderValue } from './custom' +import { CustomPlotsOrderValue, isCheckpointPlot } from './custom' import { CheckpointPlot, ComparisonPlots, diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index 95ff3c64c6..ae0e51488e 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -1,5 +1,5 @@ import { getCustomPlotId } from './collect' -import { CustomPlotsOrderValue } from './custom' +import { CustomPlotsOrderValue, isCheckpointValue } from './custom' import { splitColumnPath } from '../../experiments/columns/paths' import { pickFromColumnLikes } from '../../experiments/columns/quickPick' import { Column, ColumnType } from '../../experiments/webview/contract' @@ -53,7 +53,7 @@ export const pickCustomPlots = ( } const plotsItems = plots.map(plot => - plot.type === CustomPlotType.CHECKPOINT + isCheckpointValue(plot) ? getCheckpointPlotItem(plot.metric) : getMetricVsParamPlotItem(plot.metric, plot.param) ) diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 4c8c9310d7..09acde7015 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -32,8 +32,7 @@ import { Title } from '../../vscode/title' import { ColumnType } from '../../experiments/webview/contract' import { FILE_SEPARATOR } from '../../experiments/columns/paths' import { reorderObjectList } from '../../util/array' -import { isCheckpointPlot } from '../model/collect' -import { CustomPlotsOrderValue } from '../model/custom' +import { CustomPlotsOrderValue, isCheckpointValue } from '../model/custom' export class WebviewMessages { private readonly paths: PathsModel @@ -130,7 +129,7 @@ export class WebviewMessages { } const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { - if (value.type === CustomPlotType.CHECKPOINT) { + if (isCheckpointValue(value)) { return } return ( @@ -215,7 +214,7 @@ export class WebviewMessages { } const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { - if (value.type === CustomPlotType.CHECKPOINT) { + if (isCheckpointValue(value)) { return value.metric === metric } }) @@ -281,7 +280,7 @@ export class WebviewMessages { customPlots, 'id' ).map(plot => - isCheckpointPlot(plot) + isCheckpointValue(plot) ? { metric: buildMetricOrParamPath(ColumnType.METRICS, plot.metric), type: CustomPlotType.CHECKPOINT diff --git a/extension/src/test/suite/plots/util.ts b/extension/src/test/suite/plots/util.ts index cb11eabe11..3ac226b408 100644 --- a/extension/src/test/suite/plots/util.ts +++ b/extension/src/test/suite/plots/util.ts @@ -24,7 +24,7 @@ import { WebviewMessages } from '../../../plots/webview/messages' import { ExperimentsModel } from '../../../experiments/model' import { Experiment } from '../../../experiments/webview/contract' import { EXPERIMENT_WORKSPACE_ID } from '../../../cli/dvc/contract' -import { CustomPlotType } from '../../../plots/webview/contract' +import { isCheckpointPlot } from '../../../plots/model/custom' export const buildPlots = async ( disposer: Disposer, @@ -146,10 +146,9 @@ export const getExpectedCustomPlotsData = ( nbItemsPerRow, plots: plots.map(plot => ({ ...plot, - values: - plot.type === CustomPlotType.CHECKPOINT - ? plot.values.filter(value => domain.includes(value.group)) - : plot.values + values: isCheckpointPlot(plot) + ? plot.values.filter(value => domain.includes(value.group)) + : plot.values })) } } diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index 26c6aa923d..6f60f69a16 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -1,10 +1,9 @@ import { - CheckpointPlotData, ColorScale, CustomPlotData, - CustomPlotType, Section } from 'dvc/src/plots/webview/contract' +import { isCheckpointPlot } from 'dvc/src/plots/model/custom' import React, { useMemo, useEffect, useState } from 'react' import { useSelector } from 'react-redux' import { createMetricVsParamSpec, createCheckpointSpec } from './util' @@ -19,10 +18,6 @@ interface CustomPlotProps { id: string } -const isCheckpointPlot = (plot: CustomPlotData): plot is CheckpointPlotData => { - return plot.type === CustomPlotType.CHECKPOINT -} - const createCustomPlotSpec = ( plot: CustomPlotData | undefined, colors: ColorScale | undefined From 3bd3bc1fbd036e7fe78218a54800aa303a51c683 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Thu, 9 Mar 2023 19:20:32 -0600 Subject: [PATCH 12/40] Delete unused metrics code --- extension/src/plots/model/index.test.ts | 9 --------- extension/src/plots/model/index.ts | 18 ------------------ extension/src/plots/webview/contract.ts | 8 -------- 3 files changed, 35 deletions(-) diff --git a/extension/src/plots/model/index.test.ts b/extension/src/plots/model/index.test.ts index 9f547fba4d..c66c74ad63 100644 --- a/extension/src/plots/model/index.test.ts +++ b/extension/src/plots/model/index.test.ts @@ -45,15 +45,6 @@ describe('plotsModel', () => { jest.clearAllMocks() }) - it('should change the selectedMetrics when calling setSelectedMetrics', () => { - expect(model.getSelectedMetrics()).toStrictEqual(persistedSelectedMetrics) - - const newSelectedMetrics = ['one', 'two', 'four', 'hundred'] - model.setSelectedMetrics(newSelectedMetrics) - - expect(model.getSelectedMetrics()).toStrictEqual(newSelectedMetrics) - }) - it('should change the plotSize when calling setPlotSize', () => { expect(model.getNbItemsPerRow(Section.CUSTOM_PLOTS)).toStrictEqual( DEFAULT_NB_ITEMS_PER_ROW diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 657715e580..9ba27390d9 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -75,7 +75,6 @@ export class PlotsModel extends ModelWithPersistence { private customCheckpointPlots?: CustomCheckpointPlots private customPlots?: CustomPlot[] - private selectedMetrics?: string[] constructor( dvcRoot: string, @@ -96,11 +95,6 @@ export class PlotsModel extends ModelWithPersistence { DEFAULT_SECTION_COLLAPSED ) this.comparisonOrder = this.revive(PersistenceKey.PLOT_COMPARISON_ORDER, []) - this.selectedMetrics = this.revive( - PersistenceKey.PLOT_SELECTED_METRICS, - undefined - ) - this.customPlotsOrder = this.revive(PersistenceKey.PLOTS_CUSTOM_ORDER, []) } @@ -359,18 +353,6 @@ export class PlotsModel extends ModelWithPersistence { return this.experiments.getSelectedRevisions().map(({ label }) => label) } - public setSelectedMetrics(selectedMetrics: string[]) { - this.selectedMetrics = selectedMetrics - this.persist( - PersistenceKey.PLOT_SELECTED_METRICS, - this.getSelectedMetrics() - ) - } - - public getSelectedMetrics() { - return this.selectedMetrics - } - public setNbItemsPerRow(section: Section, nbItemsPerRow: number) { this.nbItemsPerRow[section] = nbItemsPerRow this.persist(PersistenceKey.PLOT_NB_ITEMS_PER_ROW, this.nbItemsPerRow) diff --git a/extension/src/plots/webview/contract.ts b/extension/src/plots/webview/contract.ts index d76c69c526..86cae482fa 100644 --- a/extension/src/plots/webview/contract.ts +++ b/extension/src/plots/webview/contract.ts @@ -88,14 +88,6 @@ export type CheckpointPlot = { export type CheckpointPlotData = CheckpointPlot & { yTitle: string } -export type CheckpointPlotsData = { - plots: CheckpointPlotData[] - colors: ColorScale - nbItemsPerRow: number - height: number | undefined - selectedMetrics?: string[] -} - export type MetricVsParamPlot = { id: string values: MetricVsParamPlotValues[] From 1a6e423df8166dfe6a2722a8a1d15d033556d266 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Thu, 9 Mar 2023 19:48:13 -0600 Subject: [PATCH 13/40] Add extra quick pick tests --- extension/src/plots/model/quickPick.test.ts | 64 +++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index 68948043f0..6e0b6dd415 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -132,6 +132,50 @@ describe('pickMetricAndParam', () => { expect(mockedShowError).toHaveBeenCalledTimes(1) }) + it('should end early if user does not select a param or a metric', async () => { + mockedQuickPickValue + .mockResolvedValueOnce({ + hasChildren: false, + label: 'dropout', + path: 'params:params.yaml:dropout', + type: ColumnType.PARAMS + }) + .mockResolvedValueOnce(undefined) + .mockResolvedValue(undefined) + + const noParamSelected = await pickMetricAndParam([ + { + hasChildren: false, + label: 'dropout', + path: 'params:params.yaml:dropout', + type: ColumnType.PARAMS + }, + { + hasChildren: false, + label: 'accuracy', + path: 'summary.json:accuracy', + type: ColumnType.METRICS + } + ]) + expect(noParamSelected).toBeUndefined() + + const noMetricSelected = await pickMetricAndParam([ + { + hasChildren: false, + label: 'dropout', + path: 'params:params.yaml:dropout', + type: ColumnType.PARAMS + }, + { + hasChildren: false, + label: 'accuracy', + path: 'summary.json:accuracy', + type: ColumnType.METRICS + } + ]) + expect(noMetricSelected).toBeUndefined() + }) + it('should return a metric and a param if both are selected by the user', async () => { const expectedMetric = { label: 'loss', @@ -175,6 +219,26 @@ describe('pickMetric', () => { expect(mockedShowError).toHaveBeenCalledTimes(1) }) + it('should end early if user does not select a metric', async () => { + mockedQuickPickValue.mockResolvedValue(undefined) + + const noMetricSelected = await pickMetricAndParam([ + { + hasChildren: false, + label: 'dropout', + path: 'params:params.yaml:dropout', + type: ColumnType.PARAMS + }, + { + hasChildren: false, + label: 'accuracy', + path: 'summary.json:accuracy', + type: ColumnType.METRICS + } + ]) + expect(noMetricSelected).toBeUndefined() + }) + it('should return a metric', async () => { const expectedMetric = { label: 'loss', From 359a20c4d947f5a40b0cb0671f83b642fd9422b7 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Thu, 9 Mar 2023 20:06:17 -0600 Subject: [PATCH 14/40] Update readme --- .../walkthrough/images/plots-custom.png | Bin 0 -> 31041 bytes .../walkthrough/images/plots-trends.png | Bin 27034 -> 0 bytes extension/resources/walkthrough/plots.md | 21 ++++++++++-------- 3 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 extension/resources/walkthrough/images/plots-custom.png delete mode 100644 extension/resources/walkthrough/images/plots-trends.png diff --git a/extension/resources/walkthrough/images/plots-custom.png b/extension/resources/walkthrough/images/plots-custom.png new file mode 100644 index 0000000000000000000000000000000000000000..68523646ce59a5ae0cd7f7aef6fef456fd0c2985 GIT binary patch literal 31041 zcmc$_cQBmc`!BqSB!YxAkq{(W5~74?D@Y}}D6xyxdvA-ZPDJz;y(OYsEUU!o61_!x zv>*t=qXkQlFVl& z!{Pn?{cUY+U%q^Cb91YysnOHZYiVh5aBwh|ePe5DTUAxn(a}*~UvFh)RZ&r4Vq%h; zo12=NT2N5n;^LB(mF48*WME*Bl$7-G_pCW=KfLV;$EQb_otjsfee} zL5M~x`6#%Ywyv%&T=U9%KiKKlCl#>dA$ z6;l=xf#>Ap$UoGH(&-J;?$CTA>$RY&UBJYqQ16LOnk_Zb6vycmucS?Kr z?p=(|psIb0icOe=JW5tVi<_I<^hJ>BBa<6a$TzQQABai1zORsXN`GePuP@_`LZKAh za*bsDSsuJ#;g^z^m#5bWXSplHC~IXMQpBJcU>|{V)lSebu+?ynzpNjpZsuvB?JoCH zTg|)hnu@!Qj@iuI&+zFq(-oCyy%ACz~kMdZy>y#j`V+s)^)~BSl$~=Fb z-it~Z#p;*+iOPLHbxy(Uj&)J-2IGSt<$=G%PQhSN6+izlrhX78MV2`DqvUxGuwX&e zU2OmJkrOZj&%5|^p4u6!$FVBmay|RzudF-@J@iAqo2y%mnWMBM3+4@(Szxd-Cl}B2 zd|%U*SE`A~HGHvZ7T!UKMnJkzqlTa9&xY0OlF_zZ7O8i4-A-RVz~2CW&2L? z-Dd$1)&WLoQ0A6w%ITwup5Wen4^5({&5qO}Yw;Bdg`uiD8P_?|Fg9f_)?V?R(;?48 zvuh4hb~OIzYcDHJ1eozNPzZNw&MztyI^n1(uO`S1o?eH$yg<2>o}+`o^20NTCz%9C zbklxeC+_j~s|HgnZ2S@?5+x3OPflQy?EQ6}M|G$@jW{24O3=RF?%Ni52ORAv)TLYe zY7XVP%lz^-4CXtR6oIYTx?-%Vn4bN7z#H!(00)IkBM=Im1c|nbG&9(VhueRdt-GjV z(ej;`r!Hq(1TbZ?HZ;5^dv%iJ2YAOKKXZ z^1E^hXIJE*v25HW1rC#LrR-LCnj!lpYOfa&(sJ{v9y-w_MI+rb{YTd7c=0aj=_xrK zUWfg3ndVpo7f%8jkRAV+S81Wjrt*LaDm%q7^E6oj!7joYp%~L0>m7Vo3E(et3HIdQ zUi{SXhQCx}OQ*i3eL(&W7`ZsP$oWc-<#XXx1@Ox7#h$Ov-L2++!l4=zI32n{013|Cm?dPlWu=)}dPfo0huXNo0pj`4iW*2sOQVfF zbVFweudFw_P#O^=><%^Zyvg_F7I-;p&s7C2N~F)@fiXiMZi2O89Q*xN8u1Oy;*EHz z@MvPrhC?9(4ED$E_FOAg1<_ZSopO(!^i<+uto{IU#7+uG=_|nM=QP3<@Ek>D+k9R? ztbOe+9MD3kNxs+@mlD)EXX~h|;l86sI{IWZRGd+`n zpqbGab2PMSPD<0WEcy8*^ni)4zni+yJ2Dn+b++G`&1d6jBqtPYdw2iup6j<>!=Ft3 z@j*%(Z#JS>v~N$d@Wi)Z@%620ORE7q63c-+1wYmDYoBeujgt?IoBv&=%&X#lo`mIn zmICoymOGc?cNB}pvFB$#mz!nyW!%@w`wu?=R;P1f=2Zr z0@TD`B6Ce+jjy6&q)J~6jufpsypPLmwBi6q?nU4xhma%7>DPW$meQSv!TuVqN4#V_ zn+ZE7amr?;JeXR(Q7tou?b7>G6&lZ4G%&tV18mu{7qCtz=}UJ!-`;Z+ZQ`<^p7z$? zi#Mz~FibLBP%Fmfj-++IpL^VSPGr+XYYUs}YRf&!b9eBrnItdsE9x~$%QY-RU9C^p zz~PQ1ET13(KU2G!K`G)l%9Hyw5l4ooQ!AY{RZ5XMb-7s3$ z(@uHUEEaaK$q=3IUR3p=S#kW#liNG~me^Nd9GB|lxI6Fc=9;rZk=%byd_Pl0d&Y6* z$;Zsk7o-zo?|)NH4t{BUt!qoi#A0b;^^16F>WMQJ5>#q+bSl6J*eKsGF)Fwvw3~$9 zP$_r!^+`*cNBb|nrV&Z5+foXBu&DB^PwVj3#DfHB+xuO6so_7q9PA3sZS}})i%UpR zTHm&HA!oE*E1~96B2q*>%6vjts0-jPc^;~}9&S=@mXZ?}s={aa-X&0p^;1yqg#`SZ z20j`}^CLF>K$-ryV#LVTMl9>~jfqS>ovdQT?OGkr%Y7J6!S4MRnnyb=azQGw#`ybj zJ;mdB?Qq!5=n?`_HE!px9-y_|$tLR3f;kif4To8CIF}=~9G${MxIdMQbiiQ-?!fA2 z7s+&@Zi{mQz^ifZkJ?97cO#ui^_JxMO_svtq_IG?Ut!N#7Ewo@iNrrYW74IPoP@*` zb_~t9BsJYq!#nPex0|VqTR>iaFd(IQd0WfsN8(Zvs` zuDu>eM%Tq`7H8>tI?1IIq|c+UKF(@C4}OX%h1crd24TMXCZy;G7rQVraUUjjCgf5! zlzPTP%B=>IM=>UAsoRbIVU_mHeJuD-#eC;GFa=k^-jNQ+@phK-eyBYlvby?SWuo^0 z?0SN-EYG@;8eseCaR-`aflw(7g8e@5`!_+A=05++b7eBTQiFVKnEOZy1zX2YYF-f69qy|NdwyHml2t;gcFkwS!2) zwq13)jPH#WkAR+cu4+y|ikiVkXQz~zxnf$KU+;A%%wDPqMQKEulHvkP=Zf$6hc;f47+3r`_-xo~ofx5C^&<|j~ z%yFqiP0NoOB83VqDTA9QL2kLE!?jat!utmTf;}>yW-N>Q&)#PES@eoy-Y7N&xwwho zi#&ZCW{Z*6>sMBZW6WAHKD}(0&uew+q~eTDoZ*WI#b6KyoPk@uqD4ZOE5E< z{4n3&9(dQz!}jQkrpcKWEX8l5SJ1iQ$v4MKE+-ZEq32v9%=Y%<3AHdR1x(;Y*Y!M8 zOK8Z!uH_MrG}EBZqCQMI?z^bfQP@Kb*Br;6N^`)Xb-e^Mr^ z@L5znSv~r&MAjdDP8O&+1Et4cguG8#VI=FL<|sYTH_3-+f%fTr7Ja!aG;H)69A*QJ z7?E;X7!x!>&O?LcEpWo%DG0-T{J*n&YiD`%?4xVGhBsFl9)v}SZ2GJMw|&Wz`Ty+| zwrFwc`%hA}$j^4TO$_=)40@CUjN$-sa=mj zMPImzp2k4vAdC=FFe(c@nuY%VUp#+g|KNtl?!e%)D{meDn&^cxK2WlHR4WmFCe;6Q z=sV3ef9*`ZS|c{@5I4>{XU~)}+3P-FWWRsPNReah9bXty_UJu_YXDY7zMp$hPumHy)xGSV+NLD7;LuY$eD@ zu^hr!bRw>TvER3oF+9hYupB<_OK`n5Lcu+;avDg>*fH5jMfQDe5i`*f%vibgiPNA3 z^Ox9~g!(PXW&N zgGq%sc|w;zCw{GUHaie%oLDP8#_L7{4wN^+7k~Q`T~c$W=2Mog`T#us8nM+jAI{7? zla%Z77?@~?+CsUnEyAGa_zWe@#|_ecyEkFrt?H~BrXe_UFUn}~T=cWu8^)x~*_P1U zV;Y>ty&V;qumQwOLoc??y)E~z)vdB#W?I$wDaBy`9c2e6D*f4c08)8gW`U%{Z5qUK z&I3&tN1PNu>dq~uTEk}KrUB^lY4o%%?C-I+TQEACY|&n!N8&JuDg{IWB#w+^z)Fk^ zN4j~*hdP+oB`ufHW_@Z7=HnEWjx{04cJ7Ky@b3HvH!4{KVJ@kMTT|^?+ur;6aJK;+ z;Nq-Y@r<`Qc$V1`XyDXrFlcBwBxhuj6RR+I2E?|}0L+%3yUMDjXGQ?_=iB3$Y-P~6d0Xa)sTBQWG?qyK7;^p?2Go8x*)m0SF!nbNGNq59lM;SMM^|)wdu;waESUd|qLf=5Et;h{8 z2{N|I)^*neJcGZ)=^CYViyRm7>r%VZ$JFy@X;H9C;p-(zHmK~jN>cNxGX-A+&GV<1 zX~@dypZr(xo(t!Ip6~7ZTRQPe>uix5O;{Q$Y2mXk`yV|38WnPju*5^ng-uP@o)k`s?rT*ONPV}GW|ih+C1R>&0J?SVG)RMaXmb=aba}dw zFl0-u?V|&?Wq8eIb;Xc?dU{|^vRRy~TYZi6~ah#O3#~mBg$UJGgmGhZ||4#;43;neofk3>s+f1KqT{)Hb z)c>quS3`TJ6QBXf2S0US`1KIdKCGHj{pO^TdznoIZktp0L_-}PRmdUO&EGd!;{)9_ zlU|e_iXarJrF0qC^Y-b9d5OszR-Zjm;De!b%vG3^lr{G|^#lsitY9^8%*shEq;*RVI z81l&N#;p={>B(4n=cAzZDNUG&6YU?7tMv3m{wDNa%}(7HDkWXNei0dYo|e{IH1CSJ zw25ko@s3&mX0nV`IB)Fb0dGKZ+<4^mAuuXUC~xGhNA%+!YD+_u$hZRGr$3e*95ay| z9Tz6O|9$~>6Lfxs%ZMtgJt_apH5#vLXuGFGBI%CYrz=T8cm@YS8sQL7E6_%CO<`0nx>GI)SHCx9W}n{ zA9C*G;3(w>?C`=nCvJjUW@Q(om}c-;-|7O6pBEZ0s-1HJ_8K|@H2R8&_p5MKYYRTc zMS<5}*3TX8M zzF`^b0K%>96tLF{H!4mMVDB6`VZw|Yp#jVK^)@*foRMW)S)*AtR?Km)KQ>=XV$-B; zP34ugRwEeCKj)e+yys)6M))*8A$Yj=^9uX$pC7u?LB?hc&=Ve_{;X;-jf)LSjZ zoNo{-H(dfR7@hgZe05VKjd-nZA0KCAoN+^d9daiNXqrlHS@4r$LX)z%ML05#^16}q z5$)gA=0}yX>jpl!WEb2h6*rN#zM#G**7Nh3D7!EbJk8u@QT}EBrjB$X>j@BzHc1l!79nmU+h(Ws z%hrk{2JG>B{tv=2Yh`QlD@78N4PIz3Mg{LHM;~Bs^7JXA&?b_*h>U|6ntG~8EdOnA zW@*BP;{3;x1R0bNHib{$!%OZL(Mcv#6IVCD1L00 zj7MpLE`;9;iC8Z(&LS|qy~B*Q#igsXpI$pvZB)j3Gm!fHEAxtBZtt`96|YX{H%Xdb z+A(Ln`_IB{vV}thzCT`II|e|oZayDKkJ(a}SROYZ!Apw*3V#-FeDS(pouj zf-1JSZB%_*)aypOi>tOJNbKW{c_I-Q1viIma5&&SJ~ zki->tr{dn!5#ndjFsO{s%>4ig{b8;6SIl&fjK2Dt=wYGd6H?={C?>4_ z?RYN(uAk>t3UB^|`*z}e4xBH!f?nMJQcDHmWtfn;?(WY1Xycn#O11>?j2p`5{I0Ej zDn1X0UUs7VA#h8C=2m=9m%xGWrIq4?E5#J;@Ur5|fGOFhWG%*gQs@DX5dQ3snO2~8^~gcdu=Nt8Va}()vEuB6Czi6YhyV9?W6krS;C_2n*F)` z$7hjR^E1D^7`lLT5x>NJyw8#5C0FzFrvHKrMDJO!gIDNESdfdN{Rb)R-B?13r>!B~ z*;^pvUO~cZGkt%rdsGLu1;FtDVF2@^(8)ns=2%LMf}m}t8?iismh-VJKx~6I_L#4- zt*lL4DI&)bL2lQ5BjoOzqtf6hfhi_|Cu}CD2J9BD;k|*6a?f!gAItH-Lt9>+OmMu6 z5~HVl(oR@Isx(gBYSueSk2%gh)Sgg_oCPB+?#yvvvN{sr^g-Cgb0DljGvnFl^!HCj z%#X-D49zzRuufgQr>OogroXcFo9f2Xp}F4@S4>gk&scju2d5t^-T_@+~cU9TB`PIdbPfFhiJM-|c-aoKx?Wm0lqwsWD<3?G!Ef5;vW6^> zeS>b2ZDp121_YQdat-<<8cje%%3Q10|2U;;JmFs3Q1P+2I^T4ZY02n{;`p-!<(6Ae zi*k*&6u==5US=+R@Urx$*huAXeVE3s4J9+}Gt~Qk(;q#!rYHD^;_rW;y(c!W^d0K& ztP!hP1j6S2#OZlmoqaFU-d%b=NnNvQIRDOu9-a)QM^H%~|f4+yy?#Z3HQ8H2APymqsx35?;o z08J0Z^v|!yc+Hr~Y0v&QB;(6aBr#Qdap?R(J~iOr!EnV-_sj6~lgM>I!}b)Qkc!@w z61nte9^j(cq3} zt%@%eJvPWE1sI2P`{jvGDNmvev2@o90w7V+P5jZH@M^P@8OWDD^OaN{6p)5s6<-Yc z_aL7jAZ=M7#?$8*N4_4 z4YA&@LBV_q%28B;nzmZiH0Chiv;+iZppq|!$d=?9jho8p_8c(`7wsYT=A8pdEDMUS zo`llT$>Y9trR-8I3E1j8KUz|M266$AT|5}%I|HB}u#UG)?yg@9qx<^4(qNVA4(Tiqb zDzNa(iV9;Fbf#y(qoc75!{veasqAlIV{Iu;lKn3&fN86OLqiCdB=U~zL`HPNs`~bT zCB+V1!h#=kJ75B`A?f^xx}_{y7L`>dH|(osFosB<#9F@vZ&pa)a6OIa;pHK;Vsfa> zy4Crq*KV5jAk7|cp-4Wd5ySk59h0s5cf4v{M<}(aJ22B;b3a*+&!G(XgfUSw?}@s7 zew!XjuwYZ>^3DXqT7Ta0sodu*R%KZP_$VDAYWG=B3n7z(fA{%$4pqZs65?{?3ETJHN+K>sEKKfB})h!xZE& z)j>RKqwO5=**utJ>r-exw&>`2u%pK_O6;u~cyWwcajcd>wB15yP;_6)@*0TfLjjzp$J zlb2EUhsgD7-!wALLi)kB8L!!k=DV!jQ%Gp(+Y&MMz@l2f=>vLZR|pi8-abw+t<@Gg z)@EHTm1qxSaD^@Ygb1^HwYa;UwRad@i>#846u;zANZ7(ynG%?vmN^V>+^war0$Tr-NRDp&kx)SNodxK zuU~E9|Kf|S>MfeR3#pFrCyTJi3Rwa{!kT@bTMwSQd5M)^+a3&GfQ+OURAw461&-+$ z&iB-N@$=8|KJ%V3^zq`#-}!B(#ye{8s!7xCcFjeq>hSt{K|>$na9W{%G`z}rH6qnP z0x4EDv1)>j{xDJfB$~-?u^{4}i^Q1P32bDNh*Xivf(URAJ0U$Pz2=;Zu32`3Qa$-9 zku)ditJ7KQMBQ&S^|U~|$v~BqwRv|%(_`;K{#Azp?u6LkGXP0 zG6}J^=TaTSk#{b*iff@i0Q0)hkG{>DNS(myKy^Vdl4PhV9Y^*pV5SiEgXDSFw9ir5 z5L_K^aSLDHVBD=M^L9})OS_NqMMYylJkLYr$~(W55Ayc_bBZ1M^Xa-8*Hb~q;<6u3 zk+D5g@vl>D>~I5fx_JN4hrPnWVm)&vmkyzuro}fz7BC6x3gAhHhxHFjr)=|}B<&}< zM*19}yN91KU77uaoGyOKYE;bAK1R?a#Z6pY5z}cK!^mY69ht35>8HG#6N~Wa*!8Ae z4$0)uZhQWmX~=!pBF#}mdv^*bpszyDrVm$IF>COK%$~~F`%3Ar@^|F#f{aq;aLrG* zh1T8hn@YD-e=X`oDQpSl^k6}(f>`+Q6f|chj%dK#)95RSXfx8T(Qj^?{Gsu|plXo< z8y}yMqzKdS+cRG^RbK*BJ3J%jN2}J2fSIa9qC=^kfUQ~G*Fgkx*oYI!?!d=@6#RkD zBnfR+*L{Ze@iXDm81O>rxULDan)4jJSsJ`{x9G${rBFBKWjCfB!5@Nj8_iDr)PY*lNl5tPJ_BwDarm#UJ@eU*Vd4$_$|pt1r?Q zMqXip#CM7*s#25@xVA)MI%J~m{h7BhL|Xr?CvtO~Bl#lG7h#-v@j7HYb@V>9mY7NI zI>PcJ9Bq^&)E=`2@kmi);7cnZz;ZpkfQ#{PgGbs5bMk)meHX0s#UWm%xwv_F>Dmh- zW!kX`>e!9`+lu4$v=>$+Fb3v;^L^0Jc(SQHfxX~e@buLR#Bf8YfDN``z>p7JyL)~F zWmKo6x=ZjwngbrT9GP&=-N?fXNpS%z#aBk9QC5*jGvGub5m##GGtPb?p_PjhcY>NJ z*NC_1UY(c;R(e3im=_H_I&*amNpT<6Do;Y>?Iy=FZDYPuXwqmXQk)7x7dWPzIGD&0 z?S|u71OaA6kBhy4w9m;o)o*r+&%XSLucqNxY_Yixah=eI@kqFkS;$Q0Y4?@UrJGe= zzPx!m6lX7iFn-0+l69{y5KkYCpnzgOpED3JIm|Uk;xt4)VVwW^Zm= zc8i%0&_PFcF_p{{On{p8>u#%1nUU;yzb?T<9q3gD9lSOWN2d?kcv7IxB_CiV5=CZj zBu2F5%w*2CAH#Cw2o;c}od(^JJ1=W!2o(8p*eGp_HTS=}Hm6_Vlzcd?^(c%ho zO8Vn^$b?)1R#hY-^0IO9Ces>!#n!NlLJY%J9|~lXmA+^{cV!i{eD1An7BO4C&aqhc zZoryHkQkCyAEK*&#iOUITm7*An>}G!`by2P`KvpdDY5-m#GyitpIsT_^lZW~ zmeOhDi>W0EopO{BK;9mmPQtNHcSDF;PRF1R2G?Y#m46r@0%<~7RbY=?sHM&nFnSb( zx7*2C$vpGsQCoWVrT*-)b=Gd>xR>%E@m7V zKX!nvY>N^kY3QDDpd`jn_;;ZR#s!}ZYpB1^{diRU{Aa!gy8v+zVH#dlp^$*+tRu=Uo+e(pQ(x7)3#?|8#jDKls@kp0GM%qD0 z**H!N+A!QUEw#({rbmeefpldBO>g9f=DKyL#)}Xg)$%cch%BZ~t^No)heJx3BnBoY1YYTFOxZ-LK4YX|Bp`I#3iXU?K>kz34nO$1z^0<47KFxBLGr&(%Fn?75oJu4}BQpHq@ z+C_|r<vhmsZ+iBPy35|BI z7D7V~3P}bwJPZ}9IWumGE3c-g5$j>Ld}vU2C(?ep4m_TZ_PZ=oRSh!cbvX;?ezrh) zuQ#d_C{d}&#JW*0*GhrA+*IKU5z4}hwDruza!2bDCu*}QL(+}>~Gad=|b-Ub2%WV3!`rSiv znt^NM@7&WQ*W!oj=76S=B;7&yj#(tJIkyYSDjZ+T3GRTulz-2yI_Nfx3Oxa8GOlY# z3xJoD(vVuAXy17G!KJ4j&I<=xDa0Y(_BIH2Hz5M+ky}d5YA>4$?!QAcc(r*gGvoON zhmz342xt<{8}yaGsn2dFiu_0ZEjB|q&)aW~STi+t|0Hm*#yEsD0J4)I_bec6Dd)xt z|3bwjQ0o@!@fKmfbS3#goI^~esM@G06Hn5&I4T;#S|qZl`hWd`ajbY4xg8z zm$r?ZIZy9FnM82{!^y4=*oklTd}Hn&e3)W}QoX+j!U+R_Gljs2)tY152vIni0n~7M z$T1DeabZz%Bm``s@MDg9rujsh)Ju8z9*y`JC=1xTtGI60b!hcph*XNGf6VYfi80jzm6@f+|ca z4!xJld;w(M;aw5RW?o-VuOHO;i`{i$)ZiBYHsvcR*@IiJQQxR!^ah zI{Wx>pm=ZZa;%koU=x{^>p)zJs{oe&eqI|Vt9{bkK3F`1bkk@!*G&jh*YZVgSE%o- zCitUYEV8gn*r`Ph{2-TRC$9K=IfY@}#E0H^@7i=QGRSyxum3ufvsA)KY@5Rinycye zV!aMH4VjfqZUE}nV=)rH;85~-z7U)Sp|7oX9{{0EVY^|E(>zv;Cl8wL8UXWTRS(f0 zn{xG}7@YE?mF>ai`5$?5DEkwlWSZWh=X020VDSe@zpxSKy1u@B(*S~4JTk#$nVcsI zMK^EO@}!|4njfvlMJR=>$t^u21d$Tt6^P9c&Ff`>a-9aTCIvctEtfc{3^CY?wf=(1 zc+5*>14U64R5oR;x+~#}Oh7MN;iO*A>0NUSy4pHB5#nmZ`F;6;3A5e;${b2^9AZ}` z1<5ZJPE`9aSGQ8VR$9-o>*s2Q8qY6m+RrGzc>nFS-M)1WLJ{=K-Omn_S>wM#plP6K zG|7OM3d?71Zk2EvS$Flo(p?*>Hg3drNuXB|8DXF6ZR~DT(U;a+#7rvkB7@?YxX|y5 zkLS?H8!~5)+Q4e>+zC@s$YaTslV+&deCxJ-ZQV1fmFG&o_wBpdt`|*YPbUuOwF@5;0$A5y_Yp(B$p#Q6C>yZ~4Dx z3>$GIA*D6`I`}r6hjkOas*gkvPm58=v>7>mUp1GEPptcEhR}g(X);UjABjx_55~eP zb7+*0zz@%*Ga<+X_vP`@2n0>&?`<|+(0ual#r}A-g_wcb$iHyNCGbkFb;9}_Kz9Yt zw>30uAXyetMyYA9|IEAU!q==oO^ySpJ9!xN+^-e+ui5)$w6~KLx&IZP-UC}R$&u2a zXTR=Rwo3fkm&6$Zrs+@@kKC+y%eQk=nA#h4trQ+HXDCtp)=)cAOhzA=QNqjB@sK<8 zIf6e~1TVGyu3YeSYl<`Jp4fd*Pz7q+TW_5m=o_Et(tWw|`1|a;GJz36Y)HsG%i(7& zn)bt2y^Y76V=+>^`}_rXk>}4VI|@QWus}g%NMWnIOiM*CoP~bO#th{|ZhzDLl)rJ0 z>rP5QmCL`hW6sXi45M-2QH_o1F!{XE@|Q6PeFMwQQHCjma}>i1TX)muiox}#M;~Mg zML5c|3-5E3F&6T16k8QaaTE)`667co!}E!PlV^l)jx>z76Z@Q-BA|jg@;*=(QQi8* zW~r|jG+lnOl*NEC6W`$VPK9K~G%09f_;DAM8q??E6e#|!v+%t*X~^%)@eYpubJbcXk`=>P=rUJi0Hm2@5L~fe*ur>K zZWeEqMkFHi=$~ST)6wmT8XgqKgZ%xr^A2J+qm_HwJqay3mRrmU&T-u|>OPr^azUMN z?fS;opSM0|v56ASmn}LjG>R9}i)?^BlSav#Ia6Q~a^Z~~+_92*>=+F%6v_Pk#W=`y zG;1m+iI}l6Ii{dtc&+Vs(CddRL0sG>oz3w=cZOBdTbl;%4cb(A;WNF`GAFNUXP^9( z`N57l(;{$rI4niU+wZzm{xRS2~|}^N)x0569k*1LT`JQFBAmZf>dG zkeD8hCCh7hE}GcSIKBIZwf~~l@^tQR6X@R3Yp7hMCI*`z6|Ol4{SqT;CqIDb*a2sD z%a-t>RyhM*0m&*Ilpj%n3^$faJM}C4h&NZ&s{)DL@i{KsdAIUC4;T@-Pwna%_La%P zMQJ+?2Ban5Ja4^4)>hwR;WtD(KC>^9JQ_GX7g7`<_5^gFW1Wk74Fu$t(%$i6qCwCN zNTDSVDwyu@qR>!Hp6Pm8AiuB2O?RS${7aXL!9z-{v~$*F=q4-D1CU7aVvJu~RP6bZ6ZJ(wXgkOg^sd62UGm~^@ z1T`xn^-qD1$8+r{!HJ^rdPZPr)L9Fjw@fBwa@`|H-vhPyLY7OQ%P!&zTzZAoK>;FHy#}?E zm;7IZA%Q-L0d0unPfhzn!(R$D>%CCeL6T(gg9hb}|1xt26lGqbhc3lf&I1Z+^9K$=K-|unh69-k7Q6AZ+BbQ2R9o9>fas`?Ey7f@=6<@ZN&7{2!jCSv9%CbRz*uU;STsYGZ@y^OMJF%jH zM&>r?2a?*LyHK@=tV!` z1?-@s*8V6ZRMw0@o75`fOX-$a3ktVytjh~ObPDY60gXM5gGJkT$1N}B3&Z@a;lNr| zOHWnndU#54k9}lHNsB&%C<^HN+A}im^Dc$Y^fXxbIrxmimJ^O80uBGUAOYPQ60QPW ziIr{PnK5e{hyv{>yGLqr#5Nif5LV4B1>9v-R2$%w~^jPm1XfA?Q08No{Q8S!|FwN>_SO-W-Zhs|h3SY?YrmU1+;IVZ~efdZkNNxt?jz z*r4tLG*S#CIgUriB&0hj2*`GIdNenRPU&s#jh~GMVPL0o-pDs-HnfW)R_AN{5^gJ{!TNuB9J-b2<0&>FS##gm!2h2 z$_HGoA$`S1T$ZpMyal~$+4o7_>gS0cQGbN}BvG0!2TXLeO(~tZR8RVek0`0J8FVA- zG-Kd7%#*2o(q*;9PqCrsKc*4Hi^cfsjaQSu2g^-+QzP2Erin@6Mks91wD&?T^Khy@ z3SY~D@<%X$NYlEs8`dK8-wK77CR8ZEUw`dVT+t!uQWnfXQbuKx^BZ$ph7*t*p{{*Y z?@64WuK3?;NX0)p>P>Kmc9GRuexL<2x@zt{-rv0!KEDftyqG5vpW9Ff_}Q^D-YAxX zM4RDu<2jSJzY-91)ue8fiEi-%d}Ws}*}a+9eOU?z#F=lIDh{FV(5a2&GK<;_hVntUp4v)p4YF*eMy8k5^3{?s#OWRKpv{s1b9l_KV(K@1Zf5Xh<>`eWO&VYUdrgRBB^KNP6{F z&t3;Z7f5{*XdxhW85=NXYC{x4A@U0lqQxFwmp~)0MlwlHleWye*6%}1Fn|@6Zt-)) zMfI_jIb5Yv4HT6lWS)uX8>Kya^QK+k@>$3{A7E|ExfEqru~&5KE^xFIyNZ4W<+zk( zFtwzis#b~u;yaY6&7{C*pE~1Y8`t0>i`BC28qvZN?AsCCM%c1Aj^(87VmL%>pnydP$bx+Kif2F-` z?0a*wr)L~>*XHkva@Fp(q!btskb{!Nx@A;IN>h_D?oW%v*xo7iIZ&-TQ})y83gCDj zGa?qL(;tdHED~lge@oQ$ybd@=6So4^F@2)vz-loe?BZ-UEL_bgoIu%mCjT%I1^;p#X_%Zg zO)*=(Rm;-c)`x~0o$`qbsRW_&GbRC@kyUsBEqm9U5dd(qta2>p!H!#{}B+d zoNAfkDopN9Y3QL|Ob68Tjr0900l%uqAAK0@M|>2USq%-g96?{o9ZE;Eog9%^N|D(b z+wrlxP-a%JlqxkWi1k{FJyg(fO=!1J&EqEYpU3;|**s?jNZay%PDuQ$@z#gR>K&3U z-Bq$|$_v<;zjTalCRvKt;ViGqf@rit%l-$raf)lk1<*)R585`ODCuSv5#GnsqGkUZ zlX!BLy6-esOzc-oPjSX1L^@$A!py`iu;18{tMsL|J>72I>v}h8^w3!zDBw3{KAVJ_B zPw2U(7a=jVbOnVUx(-XGoQGU;9G&=5U-!QDY3Qn!nqG7!(pnWwBxsi@109&1RA*^x z54tEc^lmG0XXjHNoNwGSJ|y1!aK_LWAXEXEna#4R{7pFp-Rm3!QM8 zUv2Y8r`lH8B}CtcoRX9zWP8zlinvy*V-Ut8-VYIN!?d95i6wT>WOBOvc6aqmI+f@t zXqmQ*rDn&t8See2xQfjdf_0Fx^Gv(}E84Uy@feEcw|e9?&{VGK667S1xTmj=tdh7U z3VD%Kmhk9x#wdNh>WhsDh$6v0q=_1CTzXw8{gOShh*sU zW6@>lbAsQ+EGLV9tdrNpR&B)Tk{y&_P%v;cazG=df36V@uYV!~ohv`;sosjG+C2;5 zvX&!LmYOiraTlzlX8*U2>P2Q$2Q=|xA#y;zA)v9?3f^ z>kUeuACt?b2qFk=!^aT4`KWy>MiEvp5y=a>f3uskf*OEdethxOZS@1Z&t{yd>FF09 zd3~o2HW=;5aVbf%VU4|eqMzcwlvUN8cm#wt!+Tv3Rb9{`!>v_GHo0hx%FLbM_Mu!- z`m|$#pg=$#`t#%9)Gf_Nnetgzk_z8Xs`u?XUtW17EJJR}-XREX-@4!p8dSqmNYrnT zFLtu=amQw^Fujuj_#QtsX8hgT$OZxdDniKlM={J_CaQ&B_+lBt(KjS)C&!^A@Y{^f zmz@)e>3DxbW`7|k%cK)&Zy{pW&^E6K`4|}b)P!NOV$l+w@_Ai#&Xi6?j3e+4r-zh zthurq$c@bK9M5J0M>@~kK6sgBTbH#EQ7UEc!ddo<&SRIz^YmJf*P~b2 zqC_%%sd+5R*`h(O!}psjb8Ny#5{5(a$wZwZ{Ot}HvrTO~B5rwkyhL;0cKPIV)4s`- zY(W3Fn#R#XP|)jxy|yyY5Q3FRb-j9Lv0vHDIMq{6>?FI7|qAk)YdPOmQs9I-G08^^xCqT3zM2OZb*SYC;Y({qiI zs++*4BN#KSeLNg9+m4;CA-~?C>%{t!HTWfg$?)OGO>nUcl!Rt#nQk=E69qdJM(B78 z?!}fy5gtjPTA9UZ;?q*S)1H>fLDuhPI+&}RUkPiQ0`-akT|;iWg(1wvWGA zb;@Idn%fyJy9$OW>zVnzwsf%tSG6 z(!?8k9fjUkw5EK?xsySceLWuD%iUJk)#N}5rJph82KwPAxpq@Re--{P6GYrt0H<9N zHg76Xkcv}0=4c=wElF8`}oLDkT+h^l~@FXJzj{ePgj~ zq5iHsby|WX5)D2U2gkY0dA&lF&f*^6n=@*-@I`EU>V~gDJ7ZkZ2w^Q2B@4SSao8p8 zNa4{hO)5=}*VJVWC|m?J-*oT=j0j@N{D3K=Gw-QXqq+H%y|BLw0l+jqyz{GP)bLVz z@oz!@dhs_2xiRm~WRf>(uxSMPtH3g*Otn6}UYg6)V`DX|q3^_T6MS6iRESqSOYgnN z!Q#2Gn(ozKSE@h!U|EC$6)ftkab`TXnIiat8U(BxmsU^Ujci5`)Bo$oO(TyR&UW`& zy588DJm3+u`=%em$KWFnqihpYC*cz~(x^@o6PaXdVh~Zv;FGwh%HR_dr^b-9bhE-N ztERJ_U}-i5|BG4eLm)8ZG4wr&ZDfgF$+MmwyPFl->X3_p#cXdb3i5R{S3v6^q(@?2p`_(T^|PQYB<_%g68AB zB}x*mpl*b5_U8A;MEEVf`I(Gs+<_yJE7qn{a|K({}?d&6ve zAoLh&xj1aEBMEFZtr8h#>i98dP_wDRK%lMn>t9Ei!8F|CJ1-3Pk5F{gUk>*-Qboo7S|5~Ke#TK$a0TjP1?UXhJC4v-7k}>z{v2mxGqvwh zWCzx-0H*}<2~HpSSZmGVN|7k z;2v8@Gif(-ilsNdn(y$J%T|z!NH$7zJfP@Skf?D=)&0=m-~sZ0(eNs_Br}bx%U4%LL5KHSAG2kbUz$U zixs0m68dUpoy1cS698SOaVhh`EN;ocTnP~!rtDwHQ!r(nQ&(Q1jyj|rapn=I!7~uk_vEV`tVDLu zTF5#3SGfL4PGW+rrym?}pVVO6{FAcTp!-p9T6i;f=Q@W@>e0KaY-dK~mICh`kv)3& z&f6$U6e{ta&}}($Oti851!sq18mGa?1{=pIvlcJ+3KMmKA7R(S4M|rpCsIf_%{h=W zeE1vo_;=zpr?HLL@3 zjm|hR8+U+g zo_O^$cHqno*r^^XWTmV5Kpce0Ji%^WJjuif+hJVp05qM4pGlb#r7wQ}U8nc%3oZ$D zcfhGc1Ao?rS7s|NqMvQFp6SGH&QRtb#rCD3(!Ly8S2hKqbyxPNc6-tXQF*f2#UWv%v@a zqEJDISwOD!-ikvFHo_04Rwqc|S%U;^rrve~)-zA=+>wM($Ga9a%}^HdNBBk? z79B}9hz&&jK4`n==a5KDiA@U7Vpjx=3Y8KLa|tKgH$Af@%=rW1ixc5p2r;4Pa2 z&)K}GV8ZGjDa>h1qv(;woygCBBEsJb4}VhZm}$L9#^ngQ-Cm1nY<(vimc^(dWCz8n z;SFIT`yezEjkB@w(vU2lu;5!-yJ6{YtP3xCvoltH&(w>iX|kHtTed}ewzDk<4d`>7e!uO>J$Y=^5OhyBT@_#+#+6`a zV}6A>*R5qe>9c)_oo$CvN$r}|nq6kS~EqCY1U)jUddHR zD_&Z=FB1#!M(nK|-(t*JP=_tQ!)p`}QmSjN z{1!p8U9NCyPqLVsCU%l))$Y@R4cbCjy&c~SVDs^Ou{BriTHF-fe>sOTjjXgZ21o53 z)ZbVPb#v!JFW*L|X#$KN%X$%%GS^Lwx!&QK%Hb-KRrwxAc!d!96G-a#Iw@r6?aVjW z;OjUeeb!6H>;SL4)fvZD(}rG&dGezTljmD!|W?!(?}6V*&VN zL9$F)G&@?t3~(>>l(pGVVZ2qNB1t{k;1YbaVc(Ju+vm<|7CL5#1gA9sZYuiU$Sq=# zNuJz)^(OI)V3RSzf}SaT^x?DAEZ+ zE%Y9`X21HfI-MNFcIt$gM@P)7Gi zrSm84s92f2JBwIJ(ZOs-_o^qv*gD!ay?Dqb+GdlNl9j^CH2Atta?)POhO4_%lv7mq zLF&)1hgbMJbk$62Gy-8l$aBQD!fDij5)OO65hloR5qh{XwE>D{1%fT~hZ?9g)f^im z?fE3h4T(RwLw>nY?luoUg^B<|k z^`meGExV&a>2%MT`-|Jv{@3J+@_4XK7q_#^oknoXtj6nbDBB3c!ITKSTrETSJ7WJ$ zI#p14ce4W=t(+TZaMNy>atWkB>D#>*?PMY^uSzVR*0aVb3%ZW3cRK`97m>>FD>kV` z8=DK9DRnMa--_!Rw-HX&XmW|#UfV}k+|sCZMRxe&0$8Jld;^#HJ4aTSY=zJD=k+2j z>csKe-5tnD+sd^MT+N~%Bk9&2G|os3D0PIm{eG!mu)JHBl3f6TuZT@rzSymy(5L7q zbmxPtY!|x=tMrtQ%hPBktIiZWUfu<~ZF+7`G9XgqCrSh|<*sfsA&@8c!c+cFFTZ03f5n|Np&9!pOm|6 z1vjt>tJ!dLOQ~@(f~cg1Y4_XUARc#8zdJeOh_+W zR|FE3(wxRhxr6*7YdsU3e-XzQadbl6U7OzHJ_PKAA&} zkuJpye7}OOQLmrV)!G|32+4Tx=OD#9^|)#O0lh}2F_oc%hbnmOHLy6C?I5g^G2b^|FzWX~d#KkekOmvJP*kNm@QsxsxeJH|!w9`pu`=?;_ zUtsLH3SyK`STQZ&N971!75wqy4!SlTKjyI0Joe*AckUwDDB{?@`y#=Ok*E(1*TBZ$ z4`Yn`R<+C{S?q=p#KX7Ty)gJd^_OK+n2&vis zm0=i>kr#Mc2C^0L&s9~my6LS@%X_>kpl}2H3ggEdQQl+N{yuBv$A_25h+bmN_U&CE zw}oXdvHiWGYr?qENc`EO?;Xo_q#${f>>(d`k2zdvsEOFoXL0st?{3+3{Kt06T_!3l zrHLFeux8!HjTNcK#2Kj#XTgq;cweWbYx!10k6>qVL#N$cU(csM*PW+&;5N7bUm&?e z@RM>bw%B8OG`Tz#inlIDG@o$R;RC-Gvjtu^=-JU}x`6RbMKGk~6C$ezwgLQ)wh){n zy{IBxZ8ckTaJaBXTN6a|l_!n{MQ>_W1M|iSvTU2NO;a18+m2gfE^x2N8yT}N7CC7SwD=v;Mh)2U`@&$H23JQ>xhu#~N(b&n2ddL3i z*JtfZN3~OJ>Z|Hx^^HP}qPs`4*ly{_Kf9qjv0xD*U(77B+gkekR3sgAKen4{hLRO2 z6n(`KB2l?;5pGjcCJB5lnmGw|ERnOT*dW6%b|dTxbSG8LkL&iL{Fg$zewb_4*S1vm zA$Gcd54Sd0nH8Y<1`xOY7IwN@QMRMC+JXpxmM0$12p>$DM>@n1P>HOuhG5CB z<3DXQQF=b$2s4d?e}{BAA6Y=Yq3XMSgqxsAyMAOPvjVn$9&VEqt0z(g;EC^>8KOq} zVD^^j7^vT+iVap|Fx^8gxqxXD!YrMs`T1U>^X@8BI>(*=Cj9nXn2jG)HSRY>LQTNx z)ze{y|0Z>>(#-XDlaIRWM6Xq5LjrjC+|X;lV&*EC<)dFqg}>W{XzT`cuUW2D!zN7q z7d!>+k5c=jGt^QW3}hgw4S30$8n!p>Vz{bU?W(yN20ew8Z6A)9$Hfdf=;D^v#;xh7 z@Zy!-sk;|1i_ZRdIM!aBY$ab__X%so>8^%~p=H z<*6}Y{$V)w$tYOTO(#IDJ(b||4c)uO?z6JfgDL3c>sR(lVVTDYibhr8Uenpy+0nxr zqn7PT*M|xc7MyW?(Z8w$pHEFo7DR1kOFUodI&#AF40;K%jRHJ!JuXeTQ3KyiD{W1m zr&-q^+1@oU%c|~7^~b5&#vO2VuW`t5eM^_B7vM(|v3YNkUE=-jMc@?V#F^TOEt*PCAK#mY zy;H%ues2Dai}>d?ffcOIcu!JRVCe?>8cNw8tmT2K4IZc3f!0%sCkBW?nX_=Lu(5#E^0P2A9``l z%2{w@5}<|_?|d0Gt!q@vS>d8)IrE_fClCssp1nPvx^2sHn^qN1EpNec`vRUiQs06l z$M+)m!ur3wh$w5!u*N^um3fzM6Slr%(Ks2;!qF77P76ri<6y}~SRNkIKgVwt#*8lY z)BgH;vu(or%l@Z`-V0`#m5eN|Hb+JLHJZ-(kJeo=D-ul-oLf5)yDPu-KEhia6KH)7 zqG$Tkr7Oqw(+j5qByIA`!RhDsUum5Hs zGHDxU2~GL{;ReqUEMC_eY^3$vU(&oWS%+#(N4#SnGR2}wIVu7T^2}~bdCPRXnBj0o zS2c*p%Vk1CV^9k6z`B*JN9bvZg7V0?VZdb~0A7~t)O!U*xK?F(-!337?NjvGQrxE@ z21gv2s;r|$1HIL`&3%9@OnYUOiV^S%6CJK#C=^iFyYfPS-QvPC3ks844W{yY@l@Fn zY%T3t4YZ>UTD6u%N2zO_r-FUf0lR%VgDKpw#4xCvcA9Q;$q13Xf1F5^kk7YTf{RCy zJzW+Vvo>_kR`*bfh}?g*mcV&hbB)wR--a%W_>M(KnriQyaZp3^;nDK|nn!s)Kgml7 zKirb-i7CHniO{|zeHH%#0(p_%3U6X>U&_uK>NgB= zJma%40fw)w(It^kE?rK}L}2Bg{VvmM_uVfqzKNqtn6jJrTDK^FLdkyV@Al?iJL&?{ zVS*_gcBXdn2Q(Al24Q#a0km8VFe=RtVwIOEREcH6nAAj~2A#9CK2}i3%&I>Opi>~}&hL=z}Zw47iJgpGmvCxkg zS`3_g3vuOQ33jFE^e@$7*-eSf8Y=iBWxUF4f-4KHXEY`b7>2uKb4YD5C zb}Mo!=ehpuDh3UFYwA@zN<&mYPv7jA%@Oi5v?-3nq*ptPMwR{oX7`QwjIXjJ)CP zqRFG?=Ql;nK-XF8&-Z+M4{ksI=2&XZUzIp98|Np`Ep|Finb4Txb=)X^YZEQ&+;^MY zL+sEtC>pjYQceiu71{ekWY#-Ore$pgJPUA=V&hm*bT^qv5Sn0ZYXaO& zjV__H`AGJJ34{Ht1|3HPIRoS2^syGLzn6Nazb$H*Kx@3&h18JKL>oQ~h>M20EMH(C zB^$b`GsBcx6j+9IvUpt$CBF)yX8~H_wvA1Bm)EAar<{DsfnU;k{S#CJbb$z{JhU@O zegP$c99DKLdN?F%^lo!_2rm(LS6U=GtH+#WUhqF{YN{UYa|i>v&%yis`j_1#BB9jv zGxuZ(4IL6D0jGqIB)PJ$=(KZB!YmaFVbeLGsDFw&&J@#jetnpvtR$sC-h?M*q(*^dg z@q;%K?^k(!vE3xdMe}@Xw>~*Cw^p9;@cSMjMx4Si>+g9G#|Q~>fx}&l&=MA{g0l6R z?--I*S0HYC>#ilXu0bwWaC(%IJs*|BKd3LzWL*L=jBga@pEW-@5$=8v^hIJ2VcS_P z8*ggA>hF8WFD2rSqapsImo@kI5*KC>O zl9=dyqycd(helN)liC*Ce3`HP>Z<FQ>jYK?o7=T*E+SZ0bl zI&Ohv^-wVN-TRcQT;F?1LwU7&lLbsU$e zpaA1G2u5p3)f7;DGMOWs2&--$?q_+s-jpXP*rPKvkhO4CLHoxsN(K3%<&G1OUK5WX7UGV2VI?pDK5UbgN%t|G`8atf5UMz5`X>lB}_Bgh2G7mBz&EU_ZLf~kVV zM^d?Q{tIRSqYLCUOF5oEx4^jOH@@G=(Fn|iyFt7waz=Zohn_xqwzX)Od+BX%pzGzk zok!KIUW=x*;@FT<&06-Jv<|roxn+?ONL^CikyxXX4qsxy)v1KNhy?{70=r-)v71s$0sZHXR|W|HIJaHmeRZrmkp0 zl|9^gVw&~G%0Sx9E6}7SO77CeXQ6*sDmH6;5(u(XQ!*7ID%WS|rbZdQqKp@^{)tmG zED-b*-?RS3Vmg$lzJw=!poYPI|HJWIk8Q2nLC@i zpZ8tkWfYm=;uX02ZZbw#f#G@I?NCXBiCaUYZJNTJ3PMCQLND9-YGz<9J6d0FSp{d& zC>i2911KEt!6`7-~Ww|!Jt34h_Tu}vW=n}l~jZb3Vp z!Qm#x{yshcn{}hotTm+5dghNmRI4q=<%g(xGHMRS1$1hx%{>H|X7)C1X2hX!;KRGq zubQ<5Uzpc^DFwBlG`qd)`9xfha|gZ+lzexk8r z!~0X)x1(kkmUmjmOV(FgGj=hVGjnsn~&fe;W4f zPiB#On!Kn#Kb7G3h0IsK>|T+~bTwO6*~A$iAuaLG74n}o>j#bcvYdH#jH^o`3GGrZ zj__jG1wVmeza~|R#%b?frnI5Dx=l!bgA=@P&W=MPK=z(tnvBqA&n~_;+;W)RNTauf zZqBG>&ySR@RP+>J?i(?k2yqunq>^3Tk)m~E1NQ|rP#K-=L*%fOdU3Eha?XO<2xZj}ohq`EHkCgfIp2oZ|MKpZeB zthR$W_quJ>eCFR{wv-q{a|7b0c`|f58JP<31MT6TB;#WebwUmcpo6c9-XGDiCGuD_ zB7YM*+GV~re~muUJ>mco$%p-o8dql#o}b)pubyjhBjCNOP4*uyZl~1V=x)RvjpteZ zt|y$My1KmNi*qr1B#-W{hyYVJy0$jz|8=MG0AI zb?+17+fc?dkv!TsUZ7x$TkzX^>*tVnzj>0aO8B~?s?_vUHvyL7wi9pYwcGjp&`HeVUC0)UVvq~ z=8}f$1LgR(vIIN-QMn4IL9a}TTZWxE93A1GUW6&m<99+>36AMT(RMSJ zgD1Tc;4(PwU`2eFrD+Ir&A_#iw%7988k*%PzFdH+*JJ!hdhu)MCeMyV>epd?Xsz2# z->XMOV6JqMI|GtnUGm1y_;n4I=tom9Fg9rV`R--t-^g&q(E3DpjmM4LfOB#U%zXfh)4wjU;`th` zSaPvNAuhm zJiMkz1{?1`|hZ_O>xqoTh$D-Gw-#?7Tdgag<$`>f(Q;fXs#f{~k{}tiGDqk0T$Xtw&}Neqa67 z*7Dhyf{?vDp~KH2!V3t^m%QmY(fT3HXk=H`dVGiFo?5wugEd|v{!G?5gosCKN%g0 z0#~A}X;hGb3w@t>T}quxO2F?67?v$s8k2l zH8no2x8yaTfZG-PJfa-jK<3@4kLpRH&^a>vv(U#4bFDI3()BYYeb$W|7Ug^Fw-O0L zTV=ZXv8*pf#g{^}M6D@OYdJS9lP7k@RLH9Yvwm%;BV0FXIW-{Ob{?#s$<>Bg zuNR)pOb9I5oLu>{tdF?nY6F#%?nSg>iSCdGL5KB;*j_cD&WKDd)Ou7=K9RjX=1~>I zIuW115RZhz-#S}k1j(uLZ6@^0rHGC%y^S(<4}5esmW1tZa(jXYhKk#smr}HlZ&fKg zb0yNb@YojFn!5L#sTXmZY-MR#kf+A)`__a&%K_yVYt#LZFY|ZgDNHP~4$UZM6Hl}B zRb7eM-tvARELNcC)uP<4W(${3yrOaojt~QSpTe2HKadEAYMFhO1a>`RCl>rDzLChT zB-wo8vI$3ZLYupB*tRz}{BOHGuAF{U+|FFa3kg+=G&S*T-!MrM)4H>={j`lb;Ljz| z$f668;wdAKmMjyp;4?<#6LqTZSnl+DGe5$n{0p9>Y)PcmnAr$LRawz(;WK2Cb>L%# zS>Bh;9wFMKqQ;L?;ku=((;h@VKaua3`2l51s8BLp&eIuS_o<1=RFKZ_CA0nM1SOup zz#X*l$fa>+py%3w)7b8~znW-|;ZDAIdThT_tsNzUhhI*uOxg^GDM+IJtcHEp4O88+ z6>nFV^0P)v<3cxclue_>0xIh@m7CVtV->eItPZKCUCPn1~e3wn%NzXeH7y{o5e}F%a z28S+GpqTX3!8cs-;5#b}ib?c6xFcHRKhsM5|lSScVK^{FJSv5Yp{Ki2$qtF t;s5-%+xl3@wR1@R`KUAW|E8-FG5`JGDb9)Cmf%W6s*0LV%H%Cx{V!om#H9cL literal 0 HcmV?d00001 diff --git a/extension/resources/walkthrough/images/plots-trends.png b/extension/resources/walkthrough/images/plots-trends.png deleted file mode 100644 index a8f83df9f3616312d9baf07400757ccc3a4675aa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27034 zcmagFbyO8^^e%pAB;*`GX*h6@t|Q$I(j{FwbazN2HMEL!iKJ2R~W!0$t6=%KZHN0s;bj|8t?yXhufHTglYal#-H?jEs!1urND2yP%+8(vz=x zdU|4FVp394j~+dG`0$~Qjt&b8i=?EaoSdAIk&&61nYy~ViHXU?!~_O|QBY9e4hjk~G&Bqh41DtBNli`7t5>ha#>O%;Gktt~Dl03WKYyN) zk9v*INY>YH(0|3CsphQYUDZzS3OBM(;toDCg;i8isY>bf;*NnFs zY;JCL)6HpUXsD{H^78UhkkAP?{ph7%s4ionD{ucu$|Nr@Pe$yKgoyme$cVmzvxS94 zQc{xDqi|DY|B8x=2M-=3SPwgDy|B|vGB-DWtms};RAiwV+}+*X+S+QYTZP*qg-YQQ2Tgw`d1$7 zSKQ{w+~!H8seM6;9!s~Lf$OWUZE*c0BHSJl8-T$aWYm=89|z9wEiP&pJf;8S6?Nqw z)MFU@cxQII*SV257$IcT5-ZG;4Od7$c+f%dHnI(LJZAC3Z_63hY6st3um>0ToXjVt z^KL_@J}XC({iMG-^EUP7J~6@wKST<^aA#l$9Js9#iQF3m8WyPn!EkAB1&;r_z$^f| z3zL%z5e2YeSn~h39{$37or{QXB<%Yp4!0lMWd7nm= zUf8iDFxpXKuP$O2t5Ll`kh2>>Bk_3lWfOoA1+)kBCW_zEO>fl9%<5rp`9L8~4499O zJ7z@!U7}`z1yX!JZCBH3joq23+#EY~!FR6Se$@gT1(Vt7zAKLGfauS()wIJzX_`up zbKc4fb2Xd@1{I!%n+?_leqnI5!}q$q!MF@zFN6GNPoI&~sIG1M4XLlY4w(q(JZbJ2 z*S7kNY7%ujw=YmLU$0+a!2G#9#fC{H%@&1}J#*nW6=}DGT!)lg&l2JvcZ-zjvjH1) z1>F5g8-4$t?Zs1}{d%X5e7jvWBqLpgPE+lgpN)i(e;RF08-Lt)XeDJ%o57^2sZo-l zO==|~-+GFCm&uN@f_Q?Dy$2_VyglbF_-b-`m0XqhCmlRwvc`8AoaQPruly$67+-od zkUi-~7_cAFJRl#xC;5##B{Dlp-TNh&EYe^Y{f>umgZV zwAh$WzlO9LhIObLIzZB#ie3*(!-;A=b^vxkjiwphEYP_-c!H>}e{1I^t>tJF^e9FX zQhzH=JDIseXRj!LD z8evYk-%f>53l&6}1&$|X(Ffa|lY2dI%S}E2TN-0wp9+;U#ZxXy!Dr=G?|dI-=Y6w& z*t-Ni^LUitZ#$ow<()C3KI0a;Aj9qU?Qb3R2tgg$KX39+%U3ot+R*4OGT*zTR20Y; z#8iZ{*;gpCXO@qM)N}Bzn>9{W_r;N7MOY@x{(fyYXHQF$pOe6J#s%~bqm23V=|fzf zA##T{nQ~uSPG(lF92j6)xfz=;UXHLvEGA|>Jhk7bCr%8?H?~%Gy@`&Wi5c>6l?7-z zN-|Ch+G}U>Ufpf(w5tB-ZdZKbq<9CmX9Tf6v};UK>0HwOa;Jkj{tiEY=nzU~$Y4yL z-y9hsYvdj6f$b?kR?Vm~?bM5+pt>Nl&iE{fqdGt7urErQ%VF6cNG@wSw+#Kc?`1|~ zoh2gLXcBu>l(=OkCOc1V;;9b!OE*WkrO7OldfnrjJecnb=IcjgPH;EtNL-K-8*5$H zj8yYdpQpa@4c zd?i6NGAABu{QuT;DT0yhO*u&W*%qEN`|IsY=+bwp-qA|#sQ9PAf@acvHjyuo<5Qsc z|9!jnbmqX6HukMp5`Q=tJLdSMT1x{tRjNTqsq_+}BFZoMVRUildwwg(fhXIP=*vGV zl6Um4DR>6wtEbq9=b*SU^aKy#lK_4uV%eZXMBP&w=47!SD{+d|pBK{NO4l;@g7;s8 zpyT720R4(xiATGwRs;V_1cC^(4wxmM>1M9(w9tf^@>qzsKmytbw23^F#{&GX=}B4# zjuh1YNa>Ww7m%%C|EG%v0t&%r3J?hkpfpUuVy|2Kep=tm(2y(ny)y?Dm}Q_e{6;ML zvAP1`s|h&JBKQAK_J3l<=l-uhi4h!K4pLYBryV1GFRfnb)z2`G7k3!Evp_p#afbWx z{;y1f(}@=4p%e1^x%)>gC3`lF@5zf4?~-Gxy3PxhJGJW<`zIG>zW&kNN-|L5F`c>D ztU%;0s_Gq^KFV<|-YrejI6DYz5f6i=^^RpyHtbGL8D*4K56@oC1@v>W+`p*{aFBJQ z_k#6XhfR0{2(SL-dx)P}4bSbdW9E|)$i(syd*Z{iBGY}Z+wcHi-U(YvVM4vytWm#^ ztC1{po5^&6wS>29Uw_)AhzM{RNqv#vI%iq6jm0-HFO@R=gpXUYD zSvvYYJ(w)uLE)FDSFhh27Su%M`wq|+FV32fv|9|)L#=DWrbCN$qjd>c-eS*QDcS(_0A!y6Z9L zyU-)(5Zr{CGFs_lH|k-n{gZR7s(aQf_O(y55)9+R6SKPLOeK5zQXhzW$z}_BuCx~! z>*NK|S^Mv}o8AcMcn4-3YFZB#I<<~Ss%i$U-BJB=0>`mqos>=d(t)04SHn|Q=7ax2 z75ruPCkK~xdac9-Pyc+=aOev=*Xk$k+Ue8FYYvK|k^Sv zr~OQl=HK1h?UOrSW+}^i+A5!IVsM^s2W&(S0(u_n2t5x4h<9g1e~sL4qqSF{lz>mq z<0bZ?*>bqXBQ~2liYd;0*o_=UV_Wfd(guSSK=czxO zN8YxQ-ZjfXDY(gd?{68Tv^cllTKkh%Mf{s=f^mD%K7k67jY-B!`#qkaPRi?c`Qa0M zmulFK6HcOh-=p=91gE3HyJqIxLLz~`!|G1Mgl)B(%;h~2@Q&HcmBw;)xgtj&#a}-q z;>A6-_xfyYX7R3E!^ek0_Uo_ieAGs3YUqQfNtww`2fH5Aj#?VWAeyN~DT!6;EQIDO z(Tb%BZg%}WwHKU!>(6tMeEhf_a|v<*y}kbfkaJ8{UHto4e=O2yq9T+h%9-hiwIoDd z^E!*S4Xb2F zw+C*(un%HcAJT<9P>f|=D7?v;WsO)sH8m;~dO_YWwAkWNNR8W*Zs@SG z)A=X34g+>Z`$!mq+6kY~1F)r3bMm@U7lESZOUI~_j0uVq6F#KT*}b0Fc^}dR6stAd zp!)f;zSR$7$}S|~L)BqlXlm>lR&5=Qz_mf(sOihs&9M$(0|u}8D(Erva+(P-G;GAj zgU^n9$>n~F(=Fxo1xr4F{Q$F5Om4oAxCOUYJjZ})e3|KlT3Xn)Sm?&&iICO;fph4^(^~6#;LLGaET^|&0B@DKL^RBt+ojSXwfai)~(Hmmkerss3v93 z{PR3i1`hZ#>uZ4Ace~D~=1WL6kK+1FalA{KL@!8Fh35>1CY{ z_c&uwWhoQcG#443(6FlJw!L;U*F}RIMWoBipPiq-;GE-_uXjW(^fcWXYWn$C$ycDb z7*K{1>3FD=%2bPGi-WnGrE;;aF2^zg#DLGFtpakQL1MrL>b}i>KJKTJgkCpK-cuJP z*Bhj~9}bI1=B5~+|M*Di>Cm^Rs-o8@dp2M>DP?T6B{ZT6Y47q`jMVsK#3q85PI2V-VGK2wK1=&`%DZ?D*+{ptrj_JARzlfgz7)nO&5 zbLr7YEN^UGH1)erwvE?{(^igRz?IwIuU2E(I{!h(M^9&UQb;;<%yen>rg{paW3g6x z(Cg`#SAV+cRY@NC;V3HYS5V(}Wp29cRW$382~HNk$uv3UwQl%0)a6{k5{qimQ5A8B z+agZMItMw0sX^KO!1zcB@2kYeJ&e(aY@7)M6Prz=cq;Xy2X?%wmic_Qt`bANy-fX{ z>B5y184O158M{8=<>kQbK?G}T`cJcM5vdM`Kx?YOS?a;4?#A!?TTe;GFT-l?V9t~F z(-EG&r8{9(c`XyOPMFaHa&)c+Yo-g+9o1oa=y{<`iSHG0L=}t5_tTKhIvkoLRo4QU zyLg*h5nm934Cohe`6L+eow(f=IyjChqf}scbINQ9k4+*;`Jrukt=a40mX5p14gnyw zT6>Z?*3_Sw$`D|g5_)v=1s?kGPvZ+qXkgNgOp(`ulRCxZD{d&(4iUrLI+%UCJoOSj z`@xU%-@;S&T0C^>2jU6HE^J@P89fP2juLrcU>aQcXk!gqc6@HJ0MO&cu2|ZLh(Xz7 z1Me0-Ttv>j=uJ0-21;VL8>|4KTO;{W{4khr8bl0XL7<6ifEc}hq^%fBx>Gy>56^rn z+@<<`<_gLq%WtqnioLA$HP=b29Q@xY_6Qz5rV4fc!uBAmDfs|qOM`YH7V!;4-P5kt z6QITL@9YM{M79qF|Fc&5U6BS+HkUBpEB%+#_|p~7B&IZ=PeFyr!u4{MIYVH}Qj)06 zlk>r}L;QB#+nlUYwYiU5>SHcHAOX}|QnSNYMf!N~ZOfbcbn z$0;U8h}~ysh3!vagu`;05=#639DiR^`ej!zxT}`b@J?1n`K&I^Ehux)*$#flKqFz z&&qkUQMPE;K4Dctq9;_BL_Jqne!Ma$6k!UMqUv@MDAn*$P4b&TNYfi!hFc#>&brBC zU~mR!XJ3mt>_v?ryfoGs4%yB<+CL1!-m~5}yQ8|-g(^E}XLu{rk4{6RNfygoK2`Kd<_{m8WzT2K_?^NW#HjW{{QS+XMxmkfzUAGn zJ&rZ-${12GCw>?AstqO{VbWMW{sg*MSo-zPs))Fa9xaGyQ~v-uU(@UGh9<2(;H_w^ z|3LR}!&`(!Cg7!dpOCF3%Pkz{rlyBV4k99Vvf{EDx3}A3tyi-@>#)lf7UjM;pkdhF z1lMta!9QC`p$k5HbyA=CnS`CK22416gn*8fz@HBa(tV(7VxVX)CTRX-jxQn}Y@z$j zCE~c3mnEUTP{zE-W;y>8^n#Ma3W7X8>Qy#Ho02@-2zz0o#k`@*y7eR(kutlFFqmPK zZOdt9ksjOwMohYngPz-36+e^e!Ur;GG6re-j|5c^2=@*Fs|B{UL7X=jZjin=1 zhb^E4DUc~%FSZKhalcK$Nt7k=vTb%Gn0*;#MBTsPd}qxu_XpD!G7M@>?)sG!Q-CP*Niu3QPMW~nB42HISBBZrM$BM;NrVYV{t!7Q`KJ_ zpvOq6CIvPduZEqDKdb%A1V`CL9%GA(XFS@F3}&^h5i>$=jjY@|Hro3pnD=uH$6zbTx3PDvhX z%2N1U>-!0&Q5$-62N-igrBC)C7#Rmg^}ZZ<$5CD*i6+NfBUY zxyEe`7_;Z*=2qtg9nfNal%rnPbioG1p+~lXALre6=@}1RlPbA7#HYLuKU;+@-loiW z>cGZW>cQsQ$n{TG$p&t2kzLe(t-0MVe&5kFIFSeGfUYI7}ed%2~vu67 zJ!AA19P*n4xWF@|B*P^`fjOt7EdmT<*bFoN<(v}F2Xy$O)mnT)N&%pc%7lA8^nDagnCc=Xz{y`0CT@OB~D@7$$xWGlCx#N|hZSVwm2%K%@~I`i>O5 zSNOsJ+_`vy&)1;pAc_Y(>yQ{+J_$(v#uf5NG%D1o5ar>hHqB}en93!&vDt5L6>7ln zQcTF+a$j|9{%%RD%CEuYlm4{dBF7UBVvwiWg0ORu6KcOG6DCzJzlSa$DSdgy){UM# z5hI2d%CpYTE2<~(#AGX9+ya$_ki+bitSTf{Rf5}iXz!`;UpB%jk+T6T_h+e#d_#aw zM*6=&#A(M--&!7Xlj|#+k|V*p$UPWy>W*Iayiw@ z3-I3%!ZqxKbG2AAqpt2q$lzH2eUZgoFNX4?e^TIEUApC;VI0aNN76Tak??ZY9x=HV?~-Q!F~Qs zOqejNZv{OycqX`hb|g)8kaMG4RldY>??Q8!w%3pVmYwq3J#x00xo>*YVf)V(0! zRBb2orx;G7(O+47WCM~fe5h1U%L~nys)I0gIg}!R?p)$_VBu`RMJ{~d(eT7cJx#=0 z1(NsX0*merHTaj3tG){RN3^Bv*q9JzeDo4!XH0S|hdT{__#;^s|91aF2UrqTgxkD) z8GAZ2cB$Lo2~+3EJL8R}OQ8%c=uOG^!Vl-|zXLCrfvwu=X6$;uYj08ed!Xe&_(Kgv zgC1CKEOmVK_3Vd*!w<`Z;tH)jH#NktWw7%C4Vm4a*FyTw-bkrIf68y*RZE z$bjN_8aw<^`z7|z$JLJ+lJEcJ>UK};t4p9Z;XOETu;*MIx+FtbOMb${fG=rT;FoXz zrrW7oyDJ_t<3D-tAHYOIxJHDDGHosCyJlfPfa|1a1C*-kyDHt+v}$$=qv@mG2f}8P zpxTwW&0>_rI~L^NU{#c#ru5uT8`>c@z}vJmGRp={jP;jGybNLPv){QTSF4OXX^D*Y zSKkyRq+|x_>L=v#FeWsE2q5yF#9im=v9lyO9&0dikb_aiRUCQDS>J83eOZV2poJd{A{1_SMsVzA)~=)f9SZtr*XEix{k`j zgR^bpJ5q$oYze2aDRZ0qHbW2qN*63wt~zd45$2O4?Oo7NTdk6&l`>R_njKubinUFv z5orth>fX9&3(3U`kR|^t@+>} zFk}6J_j`AbDsMYBpM%s6(IsJCKS}AJsJ`=b^eip|6MiE;<5;vcGJ_0ibs>;$jN06v zq0W+lr@5TF^QSg6`Ewj_Bhuau{0dM$(?ZH&+9TrC)@wxL_O(M!z=A+Jd>F2@+Fbed?^ol`M9`netidN> zOU`o{#rFwJ0$jT|=qDoe|Gd7zD^!+_9tR|^u+=E#@8zI2WD^Y}>JTsvcPOHyGoZ3w ze*|7kB2djCe8W<9T$AFYuoo!KnI5EJ;H!YlXf9_EZmc^}BNy4h!`v{TsRxC-0v-y8 zDmeZ7M5cQTUa7{OZdFI~YqP&ReRSLIt)kj2FMWfPgT%)CLvB>i9)(_5w-;|-FtfSC zn6k3o`g(mOYgJ8Z(Be{@*!&!3(^XD|!jE|0`IuU;(wN*QXHuNv_|Rj6#4?cA!fo12 zZMdps+Ni$oeb)-dv1iMqgNUCdH%KO_VAds&~@CA z!P7PamwWQ%8{gf_`}USSU%T(C-jNhMX6^?gWv1ijzKnkxI&6=f6jnoK&|pgI{UzGO zRNrhN1VwEUsbudx6Vv(u=;tPO`On*OxG2crdyhHQ`LWbqH&~E+2>jJleW=mQC=t^0 zuu(0uuwEy9K5JT2Rnx~#y9~V`N?)pUsi$rlyMqhNBNUH~2%(3}8_T)sT0Q=-<46;V z>xHYj+FEVzwU=xjvgcq{wbY*;{UhMr->^B5Z6{hjx^i1)}td&f?O(cG3nT0Y0I);Y>X7kNrq2vu_^>f zpc@Bj$v8+Kh+rKjnDF`gOA|8ORos}qO|?j)7nkvOpyN1`A7`gYWqLYh@X6D{$89n1>ET*}=e zcbsCvdI@4~qi#gye3)FJ1v`{+-+>4ReSfW^wDHWKihft~LSfk6!>w@$8}oIaKFQ$X zOe`f;E+1q$8OB>S;~wV9&n*N9R5d4seI`S z3l~_9o?UWvB&%_Ee?4l+1pQ0hR=8bx^`i2@-$w75oG^Rf)z>{_=LHur1sLk;^FV1k z%@4a(Me|{<8)PB4VKb-d`n)f0WoLTzcQ zPv*D^YupfZ2YNL^Pd_ns%jqI&5Gcc0auRJ85#0LJSsV#j9Am0LO=IjCJj~SK(ST+o zvaF}KW#!oh^kw#}3Y)d)nadK6?wNp2#}%m%B1vAM}O|HSg>_5;T*U9GC0Kv0Rff)8u2GY+N= zWfOZyWfM~V9ds*}zDw1}`|Q*i4APkG_7CUIAa6$6fWR9Do5BQ5SSXy zd72Q)92-VNwQi~ykc`YQ>j5c73?r$lXk#;`lITJPPetVQFz;1Kwg!&+=`3^jW2tg} zv^(;f?&T~zq)v~|`rx5rp`lxWfFgFo{!|2fSdWUd;t~Xm=BTJXBNsNGsB8C@?$YoKNfBNOQ-&jmTqM^tX=AONO1&&fp_&REA10 zIiEY(xbbbQ!y+Mkjx%|S!?O>C^EH+4o_^d?9&3x}$-^_rzj0FMALbD@zi3G6=^Z4h#;r6Y9w~?-oYF>Yr=GWA9`6yGDvQX%9 z1+R$%4Zq-1*Qtygub*x=CUzT8##s@8#PkuE8JeF(HzT_8^DVfYBR9-dl=fKy-_#J9 zp|Lm*;Wv(uENPzRh1ljX3yyfwb|^pkdt2pQm7Rc%9Wq}f9xWe&4b7uEf3A|oLq@DS z{KV#_aZ9Jo`aJ8Byb9EZKIh(v<5MzMi0<~XBV`xoXjJ)jBa!E7!bT?n3)1K+eMy5B z#OLw!dP{|BxDOwCvq%7!Z!R^15A}A?TK@bof=@Nb9zeK4x6;tm{|g&vm9&*7faUj4 zRZ&bzGbiTv(S&7iI%_AfO~Xf=u(aQ2i(^u12SV{bvBj=gv2VzKx6@S$J1%#Iq0zCs z@1*Q%Y=xOm!is#@+f)WwbqKf5H{-60BX?9a7yD&9KkF?9b5m8#mYTd7f8 z@~kx!i^3grT|i-;H2%l0MZVG=KeUcNho8R5dcH;Y4F(jYMcV4m&vkl3v>wvbU4N0w zBP-2Fm17)yZ*?ru?jsGR7qw#Kn4fm;v@DWkd%HH2KvI8(5AORi)^!e~dYaGyS{3%|6Wz2USv}DTO zohDn%&&|CId;SjSNcw!O{^&%pdaHBQ0805(v&Pop!;|a&cN7< z%8)mp|Ko--tAt@vN)L<%Q1)B=5!4kKLFFF0KQ=Bellwi4=`K}22H;}K1si>B&4qx> z?t_nnjVaU&)-TV$4_f-N7JY0Qr!B^2GRh4~U2r_M${b;&WH1em6fK(NRiEp4(XF`y@1eos~MKoL`2BxzU;-Z|Vm zeTcjd{T z$ov}m3nBRdS&gh{VCXHkSqwO?+r6C$ANO{Ga4A%xXQ=EX2f0;wx;TIGzGuBP^5j1~ zpK8s$dXF%xeunxC z(xWxHnbqtah~A`7N0{|v!RYb?y&tmJ3)|o(@U3+}N=K^j<_4Q~nJSRvx_$0&OprHw z_<%p&3<^7y*+;6vj2<8xX0@u%Z8#8 zQ-b#I;$$N%Kq%-#+eWX2f4ILSSGU)4NaEZObB3sNusxA(yEpWh>{7?YV7MjioYZ`I zl`@r@VuDX;hAi2F0kb-FI`A7eodF}1^HbNC2izFD$J|%JLHPW^di;T%+XjQuU}J`s ztJ(F_vejVUQocA7m51k}^s+Rq-cX9cEYum45K@ZWi@KlsY@80fYgLQ#*YBsO`xLnc zws=Y7_GZeb{koA)vhp~Nemnh<9~1CGO_Hqs^Apw$e}aE5_IHk++cj6;7c)3YrIz_f z4`2ifQ{b`9yPzR9hV(kL7sa+0tmn1SH{b(R5`y}wU4td#}A!_>fAxH$g{*8lLk~vLcXHVw5-EHV~QaJvlSAp{1Aj->Q+aynv=C@a$ zup`CM;ZDi=#=jD@7rYr}&%dH!rkHyl^U;A_kuS*-8;wpZo0{wGxE|q>*Svb91}++& z*r>i0{-JE(nI_MGsrM;Lmm7}PCs%)z^k#LJ2*u`J`&8`39@i?CXO42e{W*9ijO0_7 zWE_grQP+oAfMnpSqn6em%4mXSct=UNwPd1i~XF>o3$?>%nurc{P+u--lP$u z(c7v9MCxGagMXMh@Q#z@fs)kv{_TnE5)y7YjAI}>=?x$A)1T^y)u6%Mz933LcKp19 zUs*rl#p5Q_TA$%D5`ah9bbgtA9kP!nJ_~2uK2l#j>-VNQi%QIsvood6KX8rer*a&0 znBiRWN>hd{9$NI%l}BOF!4Bz~ZR%CFRX%W_%~(Fxme$>gnHniwSs&sB{ zF5S5I7gGQ$N2L0~q#$uy&!wSyCjcM4;S)UW`|*zM?XIuhNTU({ryO&$j~b{9b%InIQIs-kexSWn3iGwyouIhP*w3Ou%;Xyp zwUQ&Ec*0z&DS_vc(H~t}rbcT_8=iep+WVBU;Mrs;qmcxsh&S%7F^y&U1U~=N% z1HJi5LV)L~VuLb`#a%RW#K9kU{m1JzhQDm;m&(i=rad8%ulK-l^R(y}1*zvn&tksk zPm#Qs37x{)RHho3K1zN-S{(+NNJVTtBEI!ZqEjHY{Xnh72LOcjzYuRMHN)hx!P4Q6 z+(t0;iYGzO=c#ovBJtlq0}m9kwdmm4^cp7$K!cyLHXbT-*Kz*5ufs-*f7V81x_7}3 zk(_?p%=J?8-VV-v&xn4y5WA)R-+~CC*w!K*$YjgYjR1kOz=*$&vN!F?LdpW~EGG{! zZ!=YIqk*d_C_c`}@5zKhQpRN2k&eH_I!EMgvU{YyymKIK>ZpqkXiX9q{^>cFJ}Io$ z7ZY8)wB6mh(`8rX4LLrHa9q;CnnDD-2P;X~8G!;7kASj|7 zCE)8S!_Bc1tvUs$1(q$M^vJ9qs=2y9+Ml$Qb3poR|2kL*VyMqo%!|dL6w0XCZ7597 zK1BY0+q}woSXt-!(;6~1hsKy)dZ)>Z`S*I}&iSh|L|5BncfVsoXKoq&*TCW?h&H+O z#-kwP`k~osbd#ra>=&)Q%m~7l(##e}?yXT`z=&}l>Ko7CrT1SR5^E*Ti`ZDSVg!%@K4o}NX@X@+N;o!`5}rKlscH7L_0Fz@alokqg{ zIPw-w!Ef6Zn|Xmri_t|jaYWQeM8oP`JsNnu*go}L_vd+_Jg!@TJ|fX3F!vrk+q{%A zvgudv{ug8)twZ_JP1RLYv0X9H^iywnBd+jh|Gi7`KQ0Tni}TxW7cjAUeE+e|LhK5Z z5eYy#v^Gv<+u{_~`=-Mi`b44V#OwPE60z|Ya^Sh;ZEBJS=^(YVk`E=(YdN1k00bdp zrPJlBT=br)Hf>wIe0NO`B=`LLC)u7D^PtP+^)k^d{DCNiCH8>A!yzpS#gHN$>GBCH z58$`Nd`L%PB|U(&?b!MgpQ$w5n(~p6-~|o(b2640Ex;(-YASD!IOS8}vFnTQY*WzS zIZk9+cj+q4GS{_f1^{LhD z6=R(_+2soztFUL*hT@RitgjKOJlXo*GoeLfln7~Pk!xx_Z3!kjW6@U4x3t1!92`FlR2Y~QSdm@% zF)~ZYj9-u>Qa8D5oy#iqRmUq`k;HL6`h=tn7m%TgCF6ztL9xxE6*LrFPu;#Y5@RJ< zCS_6V->De$^ge4}VRc(L`*$uI$3-R++e530!Vuc$NA>hiPVib=yq&dlu$%KwA{?>njcjpNA4ZB49719PbzRPJg(t=&-lWctvF|Wr zDI8E0v$3Kyum&I{ieU%;ZI_H;XrPsxdR>RC>awDhz#(I7qrobF+1Rf*tu zr&nu!@h0$&S@T>FgH#=*(k{Zo^%_uxh7cPm|s|Yb`W{e0Abi8r58jFWZ@?Gw>`ipee(&X}! zfLQH+Yt73DRM+1bP5SjIf-wK0o6MSPGcB#ZqO)W1dP|ae|E_jLL=7)o$s}ryjcays zBwDpoyvuKxTE-4LyKZ7xK)?1JzUH&5HSV*9&v@Nl_$efzkN45cB>_pFU#q!>md+PC zAGn7{4JJ^FqPy7q8;|msr%~=Eh&#Hldzi2Cdn*Guv+wU-Bidxt!sxlLW;8z1PB>}NNnJuOHp6+V z$Cg_{i;p85JWFQxMeDycsbw8Fy9m%AY>Y|2pYdYi6E}&_IsSWZcgG0cu$<6I0*XmquT5|5 z;f*F1d;`kIkPI<;gjBz#+e!8K;q8}gV?QW=sd&gV;5Lm5MaX);b%|3~lpsf3Mbio+*vR%g81KN+h>uAz9Sv;D6m*g}KzJ+GGGup1G_;R)i0y*X>zbRt z{9nmFVhI~KgmD3d@v}3w#XI|~xt(kx63UXBwY>2?sk}D=lGS+Y2nynlO6esm8U95_ zFP?Ybk~XsmazwrP?-%loduZPoB_28m*?^1rW7B!4%z-+24rT|<&Z%f#MhLctvYvdr z`i|_|i)&C$(B1Qt4lKDVGwY9d+a)e?$?aIKO>w1 z`>s-^dANKr%ZW!*IiKicd1>`h?@wQ>Oqx_&PSEaEsij4_YEta@PFi#pZwepddp6gi zN;W|JyH+DU8oiixU4qT9AG+tk6ZYBCSfDdJ@P+nVy+~v}0@~ULX<1*X%@A^6Lg$d0 zkJEh7%}~)OsI}S}r3QrV2}$-W_#uja=7niTup#lW*cl%w@Vtb0EA*&e z*2b9Nrj6UPut{GB@N71Jq<{HFbZ@j=aNI3-#9!VCDk;5+Y~PkT0x{2V#?v>k9z#3K zY`ly3Ek&|6AeL?O7`@bwT_oFR!8)EJ5%3%*s+|%d+Txc>{sU&yf`_2jiYrUm~J>p*n+h_nRi#8%!dGJK8-@TwURh83W2gkFQtZWk(Y$x-^w$I+*2spUky4k9tW9OK$$aGKbC{N* z=B%jX`2~mvY}xB_lLei>2i@Yvx+#^aT>5kmrmy%VZ0F|Ei9*ObMDAfOz&qz9G?wCY z5ec(5*cc$hc{v8u_V^q?g!vOf9kaoV&&gi4%hLDV-UsGK-Oy&O((6a@oBlwt$!4s%B1kNaf8^}l{<8$#V%gB&1r?oT!m&WJu+ zi!eFH+nCNn*FA79mt@)QWs}l`jd@LlzOMtjweSs`n{5xWv{woj`(ab6=P%_rS2Sh| z)iEZ`C{{&eU`OTHr3F4FMSn)oR&W4cHx`8540ij@ANydhabdk;qZQAUe^5 z7(GaIA&6cwdZGn`XhF2mg%G_&3xYep|9!l#_h}x^`K+^Mt+m(Qd+qOc?I4dOIV1~E z=QHA!W$ z*il9@(FN&-IRTg>MN#zqIA-S`I@Nf4ixaQ{eqqB z8M++wrB2*p+v~m?@{#t6Q!MP;`If+>f4R-FbEHk)CGmu;M{wr84r@q}4eOZse&It@ zB5jX(kX%#sem)m*RD6uC3iBZye*e~PE?jd)E=ycIUCD<#E9O%;?u{=0Hd2i8h91Gy5A?D ze1}tOiTNk0<=DdNo;{ANXf#NDB!C#ppmq&tsi@AqF)eKxC$4e6HIn!(R>bE~DKY#` z!mX(B>E@V(g@$>4(|F&d@0@z;L$f_siF*+i%XWVnlIy+)?oThA_enFrpQ9o;>_`mT z<$Ov$gj#PNnkpyo!kq~X{bHj(_HQOrE>?<$Q^Agg*q z1y7f1T7XR8wUQCWHZCXA(NJF>gQPzB18I!3!5!_O{n^3RE%I%L8w8V89_^NNSID5{ zSIo2chTt@OG;1?M$!9i-<;F^smRgnV5|v`(E@whyl;3cGAsM>uucrfeCm+6)$$DQx zuFqaAfg=k}{)4tLgAVTZb4fI5jZ=PWlWTi?N@>@u9yElxZei0lWPpDJ*{O&mvODeO z?QG&I#i8%@o7mA&6g^X(anGoE-qsbg5&%9lc1F?Q9ZNp@kPp~j4%JmCrMXum5Ojj6 zItJtH6gCsCUR3I|Q2@%HPj=K$#4U}D-FrKlsyi#&0~T4nTF%?uQo@LjwuAQKI~_^L zF4#A#!?Av}k!{^5n}_$r&@{oMDkj?v9kYUbz}yJe5_`U%*b8+K-9zrqS(YS%G8BJI z=Gn$z(WB)pQw*z;phMF72d~LEB`!@IL%ipf4jnbyHcGEv6%yDn8A+mI@&vO2vhgdQ zxMWTN!x7@4ebl3;dzK)T=j8VRa;SQ(_pcUfRtB^>s9x;>CgYr-;i7IlN-mAO*q@sG zq9;V`!1&E0=IYoms2^%t>|TO}^cSf%eo26=Q+iZUo2Xtvs+DVp#F&Pd(gto*_D0j;N0DrfSLL!(9Rs^2l1K(it| z{i_1V&UzCiK?dK;s9V;LYx6jQaDRz@AJw*l3Ehoc`;G_9y9;D<$j~dQ=^9}!JD^1f zDSfpnFb0x>sR;Q?#`8|mJ1I|n`1jWb_u`rDzH0MFFnj0t7eTmNiPm0K?O+nlW9BfI z4MG~ani~I{N8F5Yo(avv`rGu+n%HbfIMc)V2<{1hKp#x$>Dpq#Q-F7EA}%?On^^rjtO zXT&qXWPg?uXUoa?QH~PlLcb-VC~emF(=$9Tip%O_=y$Hs1&O zx9&G|gh^7}N@JWMDCd#?mgplIx;DFWxLFfI9BlxaPqGEsE9rXG!HDrG7z_NGND;_~ z(2`cfKU=q!Ngb6+key{S!}Y3)76Y1>yM&A7Jr@h3^<}m(*p&zXX=t3#VIDEcx)1|% z5#=Hk2Yh`9#OW@E&)>(TfeO+m^FUZ}gdlXG6esc`m3hDU?-tLZC*(`$9s$mXjDQ=R zeekQYT8>6qwCopxe25?F#)J+ukytfj0VtI|&0ezE`zvL4xpIVQ)-f-J5xbwLelQbD zNT}Ly6O%J4EzomT4o)u?d%X7OtTBc-J!n zC?m$cfedBC%GXC^j6SF}Yca`LrcDxk?C5~mkb=5TOpWls(mk_jw%f1vK6*jbNrHqE zz1aMNUcMAwb|C_kA3wF zQ;*rgVjXT6xn3gBKaWUELJ+eETneY0#gPZPu-wL+aI6^aJi+i|2UXoe4@e*WyY^@XoRk|OGj`5}Cd;$g=?F)Ur#iL)Z( zG$%QL1(f#o4{CM|_5B4SZXlWJ2u|7vTE#H`0*Qvf9FGA~?M5%BnXW?H1%$@ymDjiO+aI-4AzpRzHX8@bPZ)K6uio5I|roL+r#Ao_d*&q`n=-}z(+2esNRQG{uW!;%4 z6?%)kg?CX5en%@Fm_p744>@45mdx^R-&Z*jTgo`}&CIX7zYO}z0%s>zT*UYNkM@rv zyq@^9bCpeBks^dC-=@;Rum4nUh@fr9uW{W?_Hgkoq+E5p^yQM^Z~pG$a|1RF=C4PR z$zo?NoYM)7e$UX`E8&TGZ*7qY9LTR55}@y5>>HF3ct6QMs4691y=Oxik~iYvwI+ zKSju{mryHj>we9CQeoQ5>a2z>9ZBLb|D9DBCqCBh4hvKGO8JXX{K`v*TpZ)aD?&d<&1yJd8f}RlK38zlQ z0I=swEg|6g3r%#RGEBSg+ibDL2isWARc2?|Cyua?2g^X+?=zkq$7VKNO!EWrsO1OO zV9~&K#z^x@o@+B%ulaaGy;_1nGrbY<=%w26mn^@PkwheC+-@iQt9R_V;N>%Ptu__D zU2G;>=@mi+l-l_y?5;bAZm6CTM`x1D+vOfT$Z(nM<-@_4{la5M=z(&bcVA$B^D5LL z*jdx20ZREe5_BSW?wQJ$K0eQHwIqO6&g5Lq0Ko=@eqpC@CB>Vb%KRvtjd_^a=XQxf zsP&-akktV11y@YAMJUL6vFqRw>LMMTw?P26X)e`Yzum8zp)gpW3MDE3Qzs{0?O|A; zR_z-?4(tpOqFS5ra0qgBq#n{B&k5t0@j^NpyNf#ec!RREw#w#cOl%htYUgeTO;mNq zis~0+n4KynNVEhkO^(~O;!%2dEQ;xc}3kijJqupzIo51MMX6D=xIUFFoixEbWRn78g<8X<4< z44g0KX5UH38t)+S9tAwG{P)0J=c%Xl=e@NygL(MhHG|hDcG@-)OK~>lpZl>~fq^m1 z4>S9A|9B;P&bn<0(8{!izDIQ_5%4WR`W#<)!8CAMnVG|*+B;d81WaEK|;U_hbgBIoj2>1ln)QHhXqEZ<-><*AL>qmsOXeBwa%?gNB{c z-uY`fT*LOCKh}GZIER2nUWMq*#e?4xtJdD*C*2oaqq zBkjmCLJpq^%+-aSfklNu{l~Xo>UWs(RIB}>MG<)?ZZzEAcdrz6a`lz*cXzE|etsj8 zz~WpxDJzgemK*+<+wpl;@*Dg2E{pnnTf*x76M{Q5)!PvkQ(?G_xnxLIUY*!@1nXlg zrORw-{-y{qdw`1FD?d(R7W_FGKnrh*F?w19N?WYxXtr>6Tk2)_TW|`_{j(LG$$sF; zRf$geWm-Y;lpY#V=^}?*QLnnPB}E^EjL z+pKnak`G)|s~MxEfH^zbCvTep=npP@l_`1D9Pw7%+G~4+8Iiuy5#x)Nco7!QxJx*2 zrOe)^O>sFk$I)0oM%eoPtXwY3C>3H&5BZvrQBh$)z6_V7xF!z8Pt8KV5KV z-ndj)TiERqXY&`RcDa3kDZY)n2_CDHK6u2E>nN>K#ZwS^<>%)(N&C7#O3{aryq3HsU;)JoK zISSf&@xm7Fj~a(&3^|fZzPn$*2P~?zX#8}$d5+AVqJmrfe>~5{6`yf#TVet+JgxK2=Lol*&p_5$stMG|gt#!IhtD(AHGbsG>2Boo zzJvinDG&xxi^lUMCW`;s0q}e}@MpKc|MoKk_|Wti(zFy~ci{eAbd8WhA+|g@FUi6+ zWOyQ72-9&}BG*#iutgGK(FJvs<#nCft{-oKaR3d)KuzDZ8PoP|mDcp|9l%l@NgTG@ zb2hucO+0O7kgPPG>Vt!z(-htKr(eV|Jw*nvR^{O*nwOYo$rf#4r`;6sUa ztx>MNBOCe}0;TTJx(Fi-kN_WFx2AW(!-p1MLU|c1N3_f9S5xlHD(-guqNdjUpCwu0 zZ}z6Wj9a=bwSTU)dG4&8gO7cLDakQ)Nmx~S&lM$MaQ*U@?AD=E5Vqx`>q1#RnSC4HDzIX&4U(a2SUj~KAtkqU27-OV<(4l)LaA1kXwFjHAnSbKgt?MGO zmxt8*y44RYaUFb6vo&uxU%~-x>yUOPMMp*J*vn|Cyx~7vJiB>Iwkh>`;2|A*JliEN ziU}qh&`w2ODv)YW$OfF&JZWf=-hk!f-pH{aCysfKJEFu0;{$`NSt1$V5&OgTGQ7$0 zx`wonZOFINbzU6`BLN~@^A9A~ypwla=5?ScKjE7v?UNqH7!x|?JiPEkdG?)Y1~}{a z8$)xQy7?vpCFEM`z17J(;flMDASH5sV$D;jg2JWq@of`I?Bebhci=jf#WkJjlg$5< z%jop)&3t8A#HZKTwhnW?#(zc&MR+8HEi_!kw`^Z8z{ds&M4rygIb;}N3gCAKel91t zI`nQdmVjF|B+zDoFWnd8eCYIz0~oe{eu!ZeWYqC2O%frO-sAjUa)~Vb3XmfLe7??RvYr+2)wgdn4RH!|Gi(cmw$AD>Vn1(Wk}KU7OJG zeRIB@1iWE}&zsxD87bpuUNTiKA1BWVuYPyb&cCzn;zW0~BGps{`?h zIFC}nj}tvqc*ZiTUdu9-rV*e``e2Xz6x5*-Qd<>k4=@oHb318iY&iGzivJR8G?HFU zK>V0hL9rTq{_>Y(b#dMLzyA?~O=FfG8HX{R5dr16YQcWyzpLP^>hj{-G*Hom)Uc2G z@TbFF?xQGzh_4*}@W|sw72kSdqTW1FXQ;l$N8a{2>ZXmDB7cqCLAMO3xb&N@C z7GC{TQpyKZP&cRh zKZl_}l@zk%kgL$TVte0L78xFYzs48eV}ItW^+hn@p(#qsk( zN>CB#j=ATK5X+q%8`@7V+RZF3k)W$uhW@|cor+bBU2+BB$owB_aR3ip`xhJ}_g5On zDO*=Z*6>GKhRZawNx2T%rXy}FH^|A+l=Whvj;H}YmH>gz+}KjIr1bJdYMDV+nDyNAU*bY0;wCX73UV_c3M}tL7!|yV@V#8^+DkglT~@#D|(a5 zCoaq6AEP_o0$U=Ex0`yq`+`^T8~YD+iq5YdWoGqMFAT@T|B#i)aF+_$O^i$;@So);l}y zpWsb_q8+;3&z$yUcRkRyL+s)m?^ptCc0($onc$(=i`|p6&TxzJ&@&6?ovDrb;}`jO z5wwy{--9I&?<4!?dpYoY&&R(AeX9TvJhOMDW>h;0JE_>uq~~nAEdncSf2r#eeWn_r z$X_J!>@K0~Q?>3VtV_upf~0%A_UX$x#^L8{Od`aryaB$IOI|?7P_qnY@{tNc#b7NVn43y~sUj|~G zTQT1!SVb^2@~3*0<>%bpoy~_J=lU1`rN;SbhGj1nl5Vjgd$7{%taz-A>bUR1=tNa$ z_9n4Y=`qyo+L79AyXWeNag_}ihexxb7kWlZEKAq!krs#8Vj6bnC-#rL+3yU#GX%l+ zD4)LOknH9Tu!$KSW?_tk`W?M|cnXqf_%z8Etkr$Rp9m};g60#-|JCSmP#>7P|Ki|+ zhd!c_1cLcW{!}B#^P(#iQka1@ncfS%j*>XV&tF&11aTelZVY~v*Ojvs5&!&+r^CPD zsdEmBf2?uwU7CXg8z3Jbz9SC|e*J0)E17!qILnToq|5jZ7c;z*N@o|59QLEq4UVC9 zJE=3s{H9A*nO@VX^fTT?dR|3vx(~^Q4&~UCAQ`$s1s)}ZPDSB8pZl4WY_U(o0UJhL zloRGXd>q9SG{+0ZU}KY>OjF-6XxC`hcsVwt#po1B1AosQ@^Sf%FRK zESoJ`t;*zTw5r7sxqiyg>AZ*KF~k$pDq6bB>1o6eR0@9bG3lQPc!6;-MYhPv&Qo7H z2EyYnL!9hhn{~RKMD7B=HnI{tB%`IDd2}*wKH1yYG%ZCcEW0$yPc)=Z3IR7cqNmq@ zJS#i8)o6zh#u%vB8N*Kck%K~>&(Ddw(1<>3wnI(i8bua7e76wp$#i2OIflh3*81g- z)pBiGgerD|2{&I~zGuEFF3K+t*W5;r$-xaT)eo-&1mt2yrSJVemzQCDmWslzRfd<;M?We~5T4Xs+Z`59jz!8NOtan*KEH6dm~+Z&PUIranw_g}d#Q(a zuhZV{r`qm|(Lb$^Sui)9laF&FEo8HeZxG!W_M$;*}UMiYHeDyxE_?(`W+A8nyreXOKV}K#bPs&rm3jv z=d|$I3Ldca1P@SWss(K-(ZbkQrm7gl~GiCFCRDI2gI@@APz4%&Lifi6@P z91hXgWv=FFqw_AcxMDz?yXat|Qc>66ux4HC`l|&{nDTId@hBT z`HLs_Hop1568L^(*wg%Aldf@exo#fP5|YdEf#qsQ{)i?#785hi+)2u;3z0h#;hQ~M zH1|pRu_dM0=18q?j0e(klJFaGSDp16Ny)K&qnSgzUort+8OXlylP-7klW~sa^CGGfZi)jNHN^e2uFN z&=I=fJZRwY#ix5s`zW=1@UT{(Jz%|Y8yV)Y-{!vvxO?K^!H!i1o zBEmP7l873eF%*3RO8h|ZK@-~Kvaapalx$H5Z|&ZiK)%;L9Se9ts@JPRS-#3#&1mXm zfRO*KUN7;;L*k%rZ#U~FtfYzZv4v`4OHAfc{kWEj6z&p^JDtlm?DzQfVeJKTA6w?a0=i@CWyD zMlc2{8mxcByXxLmkdj)nrV2eX(8AogJU~5H!91VZ>CzaE#lyQAfV8^&69W9*WH)wI z`TPUlNV>5qO5tLY1ySC z2cww6s@?opEzFV*^o($A-2PJZdrca+hI1`GXTiwkVH~>02VHyf+G3TwUuLoNF2K7L z`Aa`{J=Ndz%P3ECok{h_dJ`>eOt-1*=OYGz(t)V%m`QnUUelSrM}l?Yms>JGxnSr` z-INL!x+XEQ+CI*8VA!dlHVtiF^%%JbUBrD^5BFs|LZ@d1MxF?A(G;Z9MyHHcyBu5p zGC2O+Fbugyi!Mq0BI@d*=A=|6k{ij!!6c%Y*rF=<__Ih-l8I-Mc;=?Fj8oP(^X`mx z3HdM$T~|<^3L(@zmZ z3^ff?pj#s`KoyICb*KcQrXmB#mJtIsb11Q7c&tUs7s$!+w^t+Ahz~YvXlpibsmiG& zCg3swYqiEqF#=wMF7Dg97}Z|mMC z-HSc^;JXr=xh?CC^mq-r^$QF3Go9b>D((0on8?C=9HmLXGgoOZ8gSz|d)+~tNU>6U z$ekLpBrbEI{8pvz-x;3#^3opVf>$6L-dmBOYyG!jdB%+sTEBU^ATvn!`nT3eT9_OD z*vws_*i2n0F5%+1p?@gtpBJr0!N{Dng9RQO|>^-`o zZj5>sGsAxuEtEY#8}X_~$0k06-`7RHjWzn;#S7$gZ2#Y1&8E8wIXWIY&Au)Qn)@yO z&%5U~7JtD3`1+ieIM?z2`Jwsh_7u-Q

-

- Plots: Trends -

- **Images** (e.g. `.jpg` or `.svg` files) can be visualized as well. They will be rendered side by side for the selected experiments.

- Plots View Icon + Plots: Trends

-Automatically generated and updated **Trends** that show scalar [metrics] value -per epoch if [checkpoints] are enabled. +**Custom** plots are generated linear plots comparing metrics and params. A user +can choose between two types of plots, "Checkpoint Trend" and "Metric Vs Param". +"Checkpoint Trend" plots compare a chosen [metric] value per epoch if +[checkpoints] are enabled and "Metric Vs Param" plots compare a chosen metric +and param across experiments. [metrics]: https://dvc.org/doc/command-reference/metrics [checkpoints]: https://dvc.org/doc/user-guide/experiment-management/checkpoints +

+ Plots View Icon +

+ The **Plots Dashboard** can be configured and accessed from the _Plots_ and _Experiments_ side panels in the [**DVC View**](command:views.dvc-views). From c9ea4fe44abcf7b851f85e81b35e9fddc1be18ba Mon Sep 17 00:00:00 2001 From: julieg18 Date: Sat, 11 Mar 2023 15:53:56 -0600 Subject: [PATCH 15/40] Add test for creating both kinds of custom plots --- extension/src/test/suite/plots/index.test.ts | 65 +++++++++++++++----- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index 48198474cd..a4ce2c0376 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -769,23 +769,23 @@ suite('Plots Test Suite', () => { const webview = await plots.showWebview() - const mockCustomPlotOrderValue = { - metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:dropout', - type: CustomPlotType.METRIC_VS_PARAM - } - const mockPickCustomPlotType = stub( customPlotQuickPickUtil, 'pickCustomPlotType' ) - const mockGetMetricAndParam = stub( customPlotQuickPickUtil, 'pickMetricAndParam' ) + const mockGetMetric = stub(customPlotQuickPickUtil, 'pickMetric') - const firstQuickPickEvent = new Promise(resolve => + const mockMetricVsParamOrderValue = { + metric: 'metrics:summary.json:accuracy', + param: 'params:params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + } + + const pickMetricVsParamType = new Promise(resolve => mockPickCustomPlotType.onFirstCall().callsFake(() => { resolve(undefined) @@ -793,10 +793,13 @@ suite('Plots Test Suite', () => { }) ) - const secondQuickPickEvent = new Promise(resolve => - mockGetMetricAndParam.callsFake(() => { + const pickMetricVsParamOptions = new Promise(resolve => + mockGetMetricAndParam.onFirstCall().callsFake(() => { resolve(undefined) - return Promise.resolve(mockCustomPlotOrderValue) + return Promise.resolve({ + metric: mockMetricVsParamOrderValue.metric, + param: mockMetricVsParamOrderValue.param + }) }) ) @@ -808,12 +811,46 @@ suite('Plots Test Suite', () => { mockMessageReceived.fire({ type: MessageFromWebviewType.ADD_CUSTOM_PLOT }) - await firstQuickPickEvent - await secondQuickPickEvent + await pickMetricVsParamType + await pickMetricVsParamOptions + + expect(mockSetCustomPlotsOrder).to.be.calledWith([ + ...customPlotsOrderFixture, + mockMetricVsParamOrderValue + ]) + expect(mockSendTelemetryEvent).to.be.calledWith( + EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED, + undefined + ) + + const mockCheckpointsOrderValue = { + metric: 'metrics:summary.json:val_loss', + type: CustomPlotType.CHECKPOINT + } + + const pickCheckpointsType = new Promise(resolve => + mockPickCustomPlotType.onSecondCall().callsFake(() => { + resolve(undefined) + + return Promise.resolve(CustomPlotType.CHECKPOINT) + }) + ) + + const pickCheckpointOption = new Promise(resolve => + mockGetMetric.onFirstCall().callsFake(() => { + resolve(undefined) + return Promise.resolve(mockCheckpointsOrderValue.metric) + }) + ) + + mockMessageReceived.fire({ type: MessageFromWebviewType.ADD_CUSTOM_PLOT }) + + await pickCheckpointsType + await pickCheckpointOption expect(mockSetCustomPlotsOrder).to.be.calledWith([ ...customPlotsOrderFixture, - mockCustomPlotOrderValue + mockCheckpointsOrderValue ]) expect(mockSendTelemetryEvent).to.be.calledWith( EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED, From e08e30e86e5fc4046245c7dca4a4581237f70655 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Sat, 11 Mar 2023 16:17:46 -0600 Subject: [PATCH 16/40] Fix possibly broken old state --- extension/src/plots/model/index.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 9ba27390d9..e2320ef58d 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -30,7 +30,8 @@ import { CustomPlot, ColorScale, DEFAULT_HEIGHT, - DEFAULT_NB_ITEMS_PER_ROW + DEFAULT_NB_ITEMS_PER_ROW, + CustomPlotType } from '../webview/contract' import { ExperimentsOutput, @@ -184,7 +185,14 @@ export class PlotsModel extends ModelWithPersistence { } public getCustomPlotsOrder() { - return this.customPlotsOrder + return this.customPlotsOrder.map( + ({ type, ...rest }) => + ({ + ...rest, + // type is possibly undefined if state holds an older version of custom plots + type: type || CustomPlotType.METRIC_VS_PARAM + } as CustomPlotsOrderValue) + ) } public updateCustomPlotsOrder(plotsOrder: CustomPlotsOrderValue[]) { From e3eba7b065cb37b5982555c5180925d51db934e1 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Sat, 11 Mar 2023 16:26:04 -0600 Subject: [PATCH 17/40] Improve text --- extension/resources/walkthrough/live-plots.md | 6 ++---- extension/resources/walkthrough/plots.md | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/extension/resources/walkthrough/live-plots.md b/extension/resources/walkthrough/live-plots.md index e7a12f39c3..da2d3605bc 100644 --- a/extension/resources/walkthrough/live-plots.md +++ b/extension/resources/walkthrough/live-plots.md @@ -33,8 +33,6 @@ for epoch in range(NUM_EPOCHS): `DVCLive` is _optional_, and you can just append or modify plot files using any language and any tool. -💡 `Trends` section of the plots dashboard is being updated automatically based +💡 `Custom` section of the plots dashboard is being updated automatically based on the data in the table. You don't even have to manage or write any special -plot files, but you need to enable -[checkpoints](https://dvc.org/doc/user-guide/experiment-management/checkpoints) -in the project. +plot files. diff --git a/extension/resources/walkthrough/plots.md b/extension/resources/walkthrough/plots.md index a5fd9389cf..9f255d8c54 100644 --- a/extension/resources/walkthrough/plots.md +++ b/extension/resources/walkthrough/plots.md @@ -71,7 +71,7 @@ rendered side by side for the selected experiments.

Plots: Trends + alt="Plots: Custom" />

**Custom** plots are generated linear plots comparing metrics and params. A user @@ -80,7 +80,7 @@ can choose between two types of plots, "Checkpoint Trend" and "Metric Vs Param". [checkpoints] are enabled and "Metric Vs Param" plots compare a chosen metric and param across experiments. -[metrics]: https://dvc.org/doc/command-reference/metrics +[metric]: https://dvc.org/doc/command-reference/metrics [checkpoints]: https://dvc.org/doc/user-guide/experiment-management/checkpoints

From 74bc253eb5ce8dee6c46c004f642ce39660d8efc Mon Sep 17 00:00:00 2001 From: julieg18 Date: Sat, 11 Mar 2023 18:00:36 -0600 Subject: [PATCH 18/40] Refactor --- extension/src/plots/model/index.ts | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index e2320ef58d..e3a4accebe 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -461,26 +461,19 @@ export class PlotsModel extends ModelWithPersistence { plots: CustomPlot[], colors: ColorScale | undefined ): CustomPlotData[] { - if (!colors) { - return plots - .filter(plot => !isCheckpointPlot(plot)) - .map( - plot => - ({ - ...plot, - yTitle: truncateVerticalTitle( - plot.metric, - this.getNbItemsPerRow(Section.CUSTOM_PLOTS) - ) as string - } as CustomPlotData) - ) + const selectedExperimentsExist = !!colors + let filteredPlots = plots + if (!selectedExperimentsExist) { + filteredPlots = plots.filter(plot => !isCheckpointPlot(plot)) } - return plots.map( + return filteredPlots.map( plot => ({ ...plot, values: isCheckpointPlot(plot) - ? plot.values.filter(value => colors.domain.includes(value.group)) + ? plot.values.filter(value => + (colors as ColorScale).domain.includes(value.group) + ) : plot.values, yTitle: truncateVerticalTitle( plot.metric, From e62452d22e8de02c993ba98a000b84f1f61549d4 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 08:13:08 -0500 Subject: [PATCH 19/40] Delete unneeded code --- extension/src/persistence/constants.ts | 1 - extension/src/telemetry/constants.ts | 2 - extension/src/webview/contract.ts | 10 ---- .../src/plots/components/PlotsContainer.tsx | 50 +------------------ .../components/customPlots/CustomPlot.tsx | 2 +- 5 files changed, 2 insertions(+), 63 deletions(-) diff --git a/extension/src/persistence/constants.ts b/extension/src/persistence/constants.ts index e6cf900068..d793fe71f4 100644 --- a/extension/src/persistence/constants.ts +++ b/extension/src/persistence/constants.ts @@ -10,7 +10,6 @@ export enum PersistenceKey { PLOT_COMPARISON_ORDER = 'plotComparisonOrder:', PLOT_COMPARISON_PATHS_ORDER = 'plotComparisonPathsOrder', PLOT_HEIGHT = 'plotHeight', - PLOT_METRIC_ORDER = 'plotMetricOrder:', PLOT_NB_ITEMS_PER_ROW = 'plotNbItemsPerRow:', PLOTS_CUSTOM_ORDER = 'plotCustomOrder:', PLOT_SECTION_COLLAPSED = 'plotSectionCollapsed:', diff --git a/extension/src/telemetry/constants.ts b/extension/src/telemetry/constants.ts index b00f375dfc..51da6ca6c2 100644 --- a/extension/src/telemetry/constants.ts +++ b/extension/src/telemetry/constants.ts @@ -76,7 +76,6 @@ export const EventName = Object.assign( VIEWS_PLOTS_SELECT_PLOTS: 'view.plots.selectPlots', VIEWS_PLOTS_ZOOM_PLOT: 'views.plots.zoomPlot', VIEWS_REORDER_PLOTS_CUSTOM: 'views.plots.customReordered', - VIEWS_REORDER_PLOTS_METRICS: 'views.plots.metricsReordered', VIEWS_REORDER_PLOTS_TEMPLATES: 'views.plots.templatesReordered', VIEWS_SETUP_CLOSE: 'view.setup.closed', @@ -263,7 +262,6 @@ export interface IEventNamePropertyMapping { [EventName.VIEWS_PLOTS_SELECT_PLOTS]: undefined [EventName.VIEWS_PLOTS_EXPERIMENT_TOGGLE]: undefined [EventName.VIEWS_PLOTS_ZOOM_PLOT]: undefined - [EventName.VIEWS_REORDER_PLOTS_METRICS]: undefined [EventName.VIEWS_REORDER_PLOTS_CUSTOM]: undefined [EventName.VIEWS_REORDER_PLOTS_TEMPLATES]: undefined diff --git a/extension/src/webview/contract.ts b/extension/src/webview/contract.ts index ec638fa710..9e27ed29df 100644 --- a/extension/src/webview/contract.ts +++ b/extension/src/webview/contract.ts @@ -29,7 +29,6 @@ export enum MessageFromWebviewType { REORDER_COLUMNS = 'reorder-columns', REORDER_PLOTS_COMPARISON = 'reorder-plots-comparison', REORDER_PLOTS_COMPARISON_ROWS = 'reorder-plots-comparison-rows', - REORDER_PLOTS_METRICS = 'reorder-plots-metrics', REORDER_PLOTS_CUSTOM = 'reorder-plots-custom', REORDER_PLOTS_TEMPLATES = 'reorder-plots-templates', REFRESH_REVISION = 'refresh-revision', @@ -53,7 +52,6 @@ export enum MessageFromWebviewType { SET_STUDIO_SHARE_EXPERIMENTS_LIVE = 'set-studio-share-experiments-live', SHARE_EXPERIMENT_AS_BRANCH = 'share-experiment-as-branch', SHARE_EXPERIMENT_AS_COMMIT = 'share-experiment-as-commit', - TOGGLE_METRIC = 'toggle-metric', TOGGLE_PLOTS_SECTION = 'toggle-plots-section', REMOVE_CUSTOM_PLOTS = 'remove-custom-plots', REMOVE_STUDIO_TOKEN = 'remove-studio-token', @@ -160,10 +158,6 @@ export type MessageFromWebview = type: MessageFromWebviewType.REMOVE_COLUMN_SORT payload: string } - | { - type: MessageFromWebviewType.TOGGLE_METRIC - payload: string[] - } | { type: MessageFromWebviewType.REMOVE_CUSTOM_PLOTS } @@ -176,10 +170,6 @@ export type MessageFromWebview = type: MessageFromWebviewType.REORDER_PLOTS_COMPARISON_ROWS payload: string[] } - | { - type: MessageFromWebviewType.REORDER_PLOTS_METRICS - payload: string[] - } | { type: MessageFromWebviewType.REORDER_PLOTS_CUSTOM payload: string[] diff --git a/webview/src/plots/components/PlotsContainer.tsx b/webview/src/plots/components/PlotsContainer.tsx index ca09a15f59..c607065f6b 100644 --- a/webview/src/plots/components/PlotsContainer.tsx +++ b/webview/src/plots/components/PlotsContainer.tsx @@ -9,11 +9,10 @@ import { AnyAction } from '@reduxjs/toolkit' import { useDispatch, useSelector } from 'react-redux' import { Section } from 'dvc/src/plots/webview/contract' import { MessageFromWebviewType } from 'dvc/src/webview/contract' -import { PlotsPicker, PlotsPickerProps } from './PlotsPicker' import styles from './styles.module.scss' import { IconMenuItemProps } from '../../shared/components/iconMenu/IconMenuItem' import { sendMessage } from '../../shared/vscode' -import { Lines, Add, Trash } from '../../shared/components/icons' +import { Add, Trash } from '../../shared/components/icons' import { MinMaxSlider } from '../../shared/components/slider/MinMaxSlider' import { PlotsState } from '../store' import { SectionContainer } from '../../shared/components/sectionContainer/SectionContainer' @@ -24,57 +23,18 @@ export interface PlotsContainerProps { title: string nbItemsPerRow: number changeNbItemsPerRow?: (nb: number) => AnyAction - menu?: PlotsPickerProps addPlotsButton?: { onClick: () => void } removePlotsButton?: { onClick: () => void } children: React.ReactNode hasItems?: boolean } -export const SectionDescription = { - // "Custom" - [Section.CUSTOM_PLOTS]: ( - - Generated custom linear plots comparing chosen metrics and params in all - experiments in the table. - - ), - // "Images" - [Section.COMPARISON_TABLE]: ( - - Images (e.g. any .jpg, .svg, or - .png file) rendered side by side across experiments. They - should be registered as{' '} - - plots - - . - - ), - // "Data Series" - [Section.TEMPLATE_PLOTS]: ( - - Any JSON, YAML, CSV, or{' '} - TSV file(s) with data points, visualized using{' '} - - plot templates - - . Either predefined (e.g. confusion matrix, linear) or{' '} - - custom Vega-lite templates - - . - - ) -} - export const PlotsContainer: React.FC = ({ sectionCollapsed, sectionKey, title, children, nbItemsPerRow, - menu, addPlotsButton, removePlotsButton, changeNbItemsPerRow, @@ -92,14 +52,6 @@ export const PlotsContainer: React.FC = ({ const menuItems: IconMenuItemProps[] = [] - if (menu) { - menuItems.unshift({ - icon: Lines, - onClickNode: , - tooltip: 'Select Plots' - }) - } - if (addPlotsButton) { menuItems.unshift({ icon: Add, diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index 6f60f69a16..68e9826a7b 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -27,7 +27,7 @@ const createCustomPlotSpec = ( } if (isCheckpointPlot(plot)) { - return colors ? createCheckpointSpec(plot.yTitle, colors) : {} + return createCheckpointSpec(plot.yTitle, colors) } return createMetricVsParamSpec(plot.yTitle, plot.param) } From d75af27dac23ff30d32f2f377d8f20da8cff9719 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 08:21:22 -0500 Subject: [PATCH 20/40] Update readme --- extension/resources/walkthrough/plots.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extension/resources/walkthrough/plots.md b/extension/resources/walkthrough/plots.md index 9f255d8c54..37ff662135 100644 --- a/extension/resources/walkthrough/plots.md +++ b/extension/resources/walkthrough/plots.md @@ -76,9 +76,10 @@ rendered side by side for the selected experiments. **Custom** plots are generated linear plots comparing metrics and params. A user can choose between two types of plots, "Checkpoint Trend" and "Metric Vs Param". -"Checkpoint Trend" plots compare a chosen [metric] value per epoch if -[checkpoints] are enabled and "Metric Vs Param" plots compare a chosen metric -and param across experiments. + +"Metric Vs Param" plots compare a chosen metric and param across experiments. +"Checkpoint Trend" plots can compare a chosen [metric] value per epoch if +[checkpoints] are enabled. [metric]: https://dvc.org/doc/command-reference/metrics [checkpoints]: https://dvc.org/doc/user-guide/experiment-management/checkpoints From 64530ab666cc6fa1b3600ccdc5cd64de8ddb32c1 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 08:27:55 -0500 Subject: [PATCH 21/40] Revert "Delete unneeded code" This reverts commit e62452d22e8de02c993ba98a000b84f1f61549d4. --- extension/src/persistence/constants.ts | 1 + extension/src/telemetry/constants.ts | 2 + extension/src/webview/contract.ts | 10 ++++ .../src/plots/components/PlotsContainer.tsx | 50 ++++++++++++++++++- .../components/customPlots/CustomPlot.tsx | 2 +- 5 files changed, 63 insertions(+), 2 deletions(-) diff --git a/extension/src/persistence/constants.ts b/extension/src/persistence/constants.ts index d793fe71f4..e6cf900068 100644 --- a/extension/src/persistence/constants.ts +++ b/extension/src/persistence/constants.ts @@ -10,6 +10,7 @@ export enum PersistenceKey { PLOT_COMPARISON_ORDER = 'plotComparisonOrder:', PLOT_COMPARISON_PATHS_ORDER = 'plotComparisonPathsOrder', PLOT_HEIGHT = 'plotHeight', + PLOT_METRIC_ORDER = 'plotMetricOrder:', PLOT_NB_ITEMS_PER_ROW = 'plotNbItemsPerRow:', PLOTS_CUSTOM_ORDER = 'plotCustomOrder:', PLOT_SECTION_COLLAPSED = 'plotSectionCollapsed:', diff --git a/extension/src/telemetry/constants.ts b/extension/src/telemetry/constants.ts index 51da6ca6c2..b00f375dfc 100644 --- a/extension/src/telemetry/constants.ts +++ b/extension/src/telemetry/constants.ts @@ -76,6 +76,7 @@ export const EventName = Object.assign( VIEWS_PLOTS_SELECT_PLOTS: 'view.plots.selectPlots', VIEWS_PLOTS_ZOOM_PLOT: 'views.plots.zoomPlot', VIEWS_REORDER_PLOTS_CUSTOM: 'views.plots.customReordered', + VIEWS_REORDER_PLOTS_METRICS: 'views.plots.metricsReordered', VIEWS_REORDER_PLOTS_TEMPLATES: 'views.plots.templatesReordered', VIEWS_SETUP_CLOSE: 'view.setup.closed', @@ -262,6 +263,7 @@ export interface IEventNamePropertyMapping { [EventName.VIEWS_PLOTS_SELECT_PLOTS]: undefined [EventName.VIEWS_PLOTS_EXPERIMENT_TOGGLE]: undefined [EventName.VIEWS_PLOTS_ZOOM_PLOT]: undefined + [EventName.VIEWS_REORDER_PLOTS_METRICS]: undefined [EventName.VIEWS_REORDER_PLOTS_CUSTOM]: undefined [EventName.VIEWS_REORDER_PLOTS_TEMPLATES]: undefined diff --git a/extension/src/webview/contract.ts b/extension/src/webview/contract.ts index 9e27ed29df..ec638fa710 100644 --- a/extension/src/webview/contract.ts +++ b/extension/src/webview/contract.ts @@ -29,6 +29,7 @@ export enum MessageFromWebviewType { REORDER_COLUMNS = 'reorder-columns', REORDER_PLOTS_COMPARISON = 'reorder-plots-comparison', REORDER_PLOTS_COMPARISON_ROWS = 'reorder-plots-comparison-rows', + REORDER_PLOTS_METRICS = 'reorder-plots-metrics', REORDER_PLOTS_CUSTOM = 'reorder-plots-custom', REORDER_PLOTS_TEMPLATES = 'reorder-plots-templates', REFRESH_REVISION = 'refresh-revision', @@ -52,6 +53,7 @@ export enum MessageFromWebviewType { SET_STUDIO_SHARE_EXPERIMENTS_LIVE = 'set-studio-share-experiments-live', SHARE_EXPERIMENT_AS_BRANCH = 'share-experiment-as-branch', SHARE_EXPERIMENT_AS_COMMIT = 'share-experiment-as-commit', + TOGGLE_METRIC = 'toggle-metric', TOGGLE_PLOTS_SECTION = 'toggle-plots-section', REMOVE_CUSTOM_PLOTS = 'remove-custom-plots', REMOVE_STUDIO_TOKEN = 'remove-studio-token', @@ -158,6 +160,10 @@ export type MessageFromWebview = type: MessageFromWebviewType.REMOVE_COLUMN_SORT payload: string } + | { + type: MessageFromWebviewType.TOGGLE_METRIC + payload: string[] + } | { type: MessageFromWebviewType.REMOVE_CUSTOM_PLOTS } @@ -170,6 +176,10 @@ export type MessageFromWebview = type: MessageFromWebviewType.REORDER_PLOTS_COMPARISON_ROWS payload: string[] } + | { + type: MessageFromWebviewType.REORDER_PLOTS_METRICS + payload: string[] + } | { type: MessageFromWebviewType.REORDER_PLOTS_CUSTOM payload: string[] diff --git a/webview/src/plots/components/PlotsContainer.tsx b/webview/src/plots/components/PlotsContainer.tsx index c607065f6b..ca09a15f59 100644 --- a/webview/src/plots/components/PlotsContainer.tsx +++ b/webview/src/plots/components/PlotsContainer.tsx @@ -9,10 +9,11 @@ import { AnyAction } from '@reduxjs/toolkit' import { useDispatch, useSelector } from 'react-redux' import { Section } from 'dvc/src/plots/webview/contract' import { MessageFromWebviewType } from 'dvc/src/webview/contract' +import { PlotsPicker, PlotsPickerProps } from './PlotsPicker' import styles from './styles.module.scss' import { IconMenuItemProps } from '../../shared/components/iconMenu/IconMenuItem' import { sendMessage } from '../../shared/vscode' -import { Add, Trash } from '../../shared/components/icons' +import { Lines, Add, Trash } from '../../shared/components/icons' import { MinMaxSlider } from '../../shared/components/slider/MinMaxSlider' import { PlotsState } from '../store' import { SectionContainer } from '../../shared/components/sectionContainer/SectionContainer' @@ -23,18 +24,57 @@ export interface PlotsContainerProps { title: string nbItemsPerRow: number changeNbItemsPerRow?: (nb: number) => AnyAction + menu?: PlotsPickerProps addPlotsButton?: { onClick: () => void } removePlotsButton?: { onClick: () => void } children: React.ReactNode hasItems?: boolean } +export const SectionDescription = { + // "Custom" + [Section.CUSTOM_PLOTS]: ( + + Generated custom linear plots comparing chosen metrics and params in all + experiments in the table. + + ), + // "Images" + [Section.COMPARISON_TABLE]: ( + + Images (e.g. any .jpg, .svg, or + .png file) rendered side by side across experiments. They + should be registered as{' '} + + plots + + . + + ), + // "Data Series" + [Section.TEMPLATE_PLOTS]: ( + + Any JSON, YAML, CSV, or{' '} + TSV file(s) with data points, visualized using{' '} + + plot templates + + . Either predefined (e.g. confusion matrix, linear) or{' '} + + custom Vega-lite templates + + . + + ) +} + export const PlotsContainer: React.FC = ({ sectionCollapsed, sectionKey, title, children, nbItemsPerRow, + menu, addPlotsButton, removePlotsButton, changeNbItemsPerRow, @@ -52,6 +92,14 @@ export const PlotsContainer: React.FC = ({ const menuItems: IconMenuItemProps[] = [] + if (menu) { + menuItems.unshift({ + icon: Lines, + onClickNode: , + tooltip: 'Select Plots' + }) + } + if (addPlotsButton) { menuItems.unshift({ icon: Add, diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index 68e9826a7b..6f60f69a16 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -27,7 +27,7 @@ const createCustomPlotSpec = ( } if (isCheckpointPlot(plot)) { - return createCheckpointSpec(plot.yTitle, colors) + return colors ? createCheckpointSpec(plot.yTitle, colors) : {} } return createMetricVsParamSpec(plot.yTitle, plot.param) } From 8d6b3128c93fc99f29d4a169bab90219b2cfcf85 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 08:29:53 -0500 Subject: [PATCH 22/40] Delete unneeded code --- webview/src/plots/components/customPlots/CustomPlot.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index 6f60f69a16..68e9826a7b 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -27,7 +27,7 @@ const createCustomPlotSpec = ( } if (isCheckpointPlot(plot)) { - return colors ? createCheckpointSpec(plot.yTitle, colors) : {} + return createCheckpointSpec(plot.yTitle, colors) } return createMetricVsParamSpec(plot.yTitle, plot.param) } From 141c1ea2e9974a9a3bda0f87dfe8ae4a25919f16 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 10:32:49 -0500 Subject: [PATCH 23/40] Consolidate CustomPlotOrderValue and CustomPlot --- extension/src/plots/model/collect.test.ts | 85 +++------------ extension/src/plots/model/collect.ts | 34 +++--- extension/src/plots/model/custom.ts | 24 +++-- extension/src/plots/model/index.ts | 10 +- extension/src/plots/model/quickPick.test.ts | 5 +- extension/src/plots/model/quickPick.ts | 13 +-- extension/src/plots/webview/contract.ts | 16 +-- extension/src/plots/webview/messages.ts | 101 +++++++++--------- .../test/fixtures/expShow/base/customPlots.ts | 36 ++++--- extension/src/test/suite/plots/index.test.ts | 2 + webview/src/plots/components/App.test.tsx | 35 +++--- .../components/customPlots/CustomPlot.tsx | 2 +- .../src/plots/components/customPlots/util.ts | 3 +- webview/src/plots/components/plotDataStore.ts | 3 +- webview/src/plots/hooks/useGetPlot.ts | 3 +- 15 files changed, 163 insertions(+), 209 deletions(-) diff --git a/extension/src/plots/model/collect.test.ts b/extension/src/plots/model/collect.test.ts index 96db2511de..1b08078f07 100644 --- a/extension/src/plots/model/collect.test.ts +++ b/extension/src/plots/model/collect.test.ts @@ -5,8 +5,8 @@ import { collectData, collectTemplates, collectOverrideRevisionDetails, - collectCustomPlotsData, - collectCustomCheckpointPlotData + collectCustomPlots, + collectCustomCheckpointPlots } from './collect' import plotsDiffFixture from '../../test/fixtures/plotsDiff/output' import customPlotsFixture, { @@ -18,13 +18,7 @@ import { EXPERIMENT_WORKSPACE_ID } from '../../cli/dvc/contract' import { sameContents } from '../../util/array' -import { - CustomPlot, - CustomPlotData, - CustomPlotType, - MetricVsParamPlotData, - TemplatePlot -} from '../webview/contract' +import { CustomPlot, TemplatePlot } from '../webview/contract' import { getCLICommitId } from '../../test/fixtures/plotsDiff/util' import expShowFixture from '../../test/fixtures/expShow/base/output' import modifiedFixture from '../../test/fixtures/expShow/modified/output' @@ -35,67 +29,21 @@ const logsLossPath = join('logs', 'loss.tsv') const logsLossPlot = (plotsDiffFixture[logsLossPath][0] || {}) as TemplatePlot -describe('collectCustomPlotsData', () => { +describe('collectCustomPlots', () => { it('should return the expected data from the test fixture', () => { const expectedOutput: CustomPlot[] = customPlotsFixture.plots.map( - ({ type, metric, id, values, ...plot }: CustomPlotData) => - type === CustomPlotType.CHECKPOINT - ? { - id, - metric, - type, - values - } - : { - id, - metric, - param: (plot as MetricVsParamPlotData).param, - type, - values - } + ({ type, metric, id, values, param }) => + ({ + id, + metric, + param, + type, + values + } as CustomPlot) ) - const data = collectCustomPlotsData( + const data = collectCustomPlots( customPlotsOrderFixture, - { - 'summary.json:accuracy': { - id: 'custom-summary.json:accuracy', - metric: 'summary.json:accuracy', - type: CustomPlotType.CHECKPOINT, - values: [ - { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, - { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, - { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, - { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, - { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, - { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, - { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, - { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, - { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, - { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, - { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, - { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } - ] - }, - 'summary.json:loss': { - id: 'custom-summary.json:loss', - metric: 'summary.json:loss', - type: CustomPlotType.CHECKPOINT, - values: [ - { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, - { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, - { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, - { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, - { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, - { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, - { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, - { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, - { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, - { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, - { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, - { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } - ] - } - }, + checkpointPlotsFixture, [ { id: '12345', @@ -135,6 +83,7 @@ describe('collectCustomPlotsData', () => { } ] ) + expect(data).toStrictEqual(expectedOutput) }) }) @@ -204,13 +153,13 @@ describe('collectData', () => { describe('collectCustomCheckpointPlotsData', () => { it('should return the expected data from the test fixture', () => { - const data = collectCustomCheckpointPlotData(expShowFixture) + const data = collectCustomCheckpointPlots(expShowFixture) expect(data).toStrictEqual(checkpointPlotsFixture) }) it('should provide a continuous series for a modified experiment', () => { - const data = collectCustomCheckpointPlotData(modifiedFixture) + const data = collectCustomCheckpointPlots(modifiedFixture) for (const { values } of Object.values(data)) { const initialExperiment = values.filter( diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 6cbf399732..b1a3dae62e 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -3,7 +3,11 @@ import get from 'lodash.get' import { TopLevelSpec } from 'vega-lite' import { VisualizationSpec } from 'react-vega' import { CustomCheckpointPlots } from '.' -import { CustomPlotsOrderValue, isCheckpointValue } from './custom' +import { + CHECKPOINTS_PARAM, + CustomPlotsOrderValue, + isCheckpointValue +} from './custom' import { getRevisionFirstThreeColumns } from './util' import { ColorScale, @@ -215,12 +219,10 @@ const collectFromExperimentsObject = ( } } -export const getCustomPlotId = (plot: CustomPlotsOrderValue) => - isCheckpointValue(plot) - ? `custom-${plot.metric}` - : `custom-${plot.metric}-${plot.param}` +export const getCustomPlotId = (metric: string, param = CHECKPOINTS_PARAM) => + `custom-${metric}-${param}` -export const collectCustomCheckpointPlotData = ( +export const collectCustomCheckpointPlots = ( data: ExperimentsOutput ): CustomCheckpointPlots => { const acc = { @@ -246,11 +248,9 @@ export const collectCustomCheckpointPlotData = ( for (const [key, value] of acc.plots.entries()) { const decodedMetric = decodeColumn(key) plotsData[decodedMetric] = { - id: getCustomPlotId({ - metric: decodedMetric, - type: CustomPlotType.CHECKPOINT - }), + id: getCustomPlotId(decodedMetric), metric: decodedMetric, + param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT, values: value } @@ -259,7 +259,7 @@ export const collectCustomCheckpointPlotData = ( return plotsData } -const collectMetricVsParamPlotData = ( +const collectMetricVsParamPlot = ( metric: string, param: string, experiments: Experiment[] @@ -267,11 +267,7 @@ const collectMetricVsParamPlotData = ( const splitUpMetricPath = splitColumnPath(metric) const splitUpParamPath = splitColumnPath(param) const plotData: MetricVsParamPlot = { - id: getCustomPlotId({ - metric, - param, - type: CustomPlotType.METRIC_VS_PARAM - }), + id: getCustomPlotId(metric, param), metric: metric.slice(ColumnType.METRICS.length + 1), param: param.slice(ColumnType.PARAMS.length + 1), type: CustomPlotType.METRIC_VS_PARAM, @@ -294,19 +290,19 @@ const collectMetricVsParamPlotData = ( return plotData } -export const collectCustomPlotsData = ( +export const collectCustomPlots = ( plotsOrderValues: CustomPlotsOrderValue[], checkpointPlots: CustomCheckpointPlots, experiments: Experiment[] ): CustomPlot[] => { return plotsOrderValues .map((plotOrderValue): CustomPlot => { - if (isCheckpointValue(plotOrderValue)) { + if (isCheckpointValue(plotOrderValue.type)) { const { metric } = plotOrderValue return checkpointPlots[metric.slice(ColumnType.METRICS.length + 1)] } const { metric, param } = plotOrderValue - return collectMetricVsParamPlotData(metric, param, experiments) + return collectMetricVsParamPlot(metric, param, experiments) }) .filter(Boolean) } diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts index 80a2a411b5..410198456b 100644 --- a/extension/src/plots/model/custom.ts +++ b/extension/src/plots/model/custom.ts @@ -1,21 +1,25 @@ import { CheckpointPlot, CustomPlot, CustomPlotType } from '../webview/contract' -type CheckpointValue = { - type: CustomPlotType.CHECKPOINT - metric: string -} +export const CHECKPOINTS_PARAM = 'epoch' -type MetricVsParamValue = { - type: CustomPlotType.METRIC_VS_PARAM +export type CustomPlotsOrderValue = { + type: CustomPlotType.METRIC_VS_PARAM | CustomPlotType.CHECKPOINT metric: string param: string } -export type CustomPlotsOrderValue = CheckpointValue | MetricVsParamValue - export const isCheckpointValue = ( - value: CustomPlotsOrderValue -): value is CheckpointValue => value.type === CustomPlotType.CHECKPOINT + type: CustomPlotType.CHECKPOINT | CustomPlotType.METRIC_VS_PARAM +) => type === CustomPlotType.CHECKPOINT export const isCheckpointPlot = (plot: CustomPlot): plot is CheckpointPlot => plot.type === CustomPlotType.CHECKPOINT + +export const doesCustomPlotAlreadyExist = ( + order: CustomPlotsOrderValue[], + metric: string, + param = CHECKPOINTS_PARAM +) => + order.some(value => { + return value.param === param && value.metric === metric + }) diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index e3a4accebe..6fb137b79e 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -10,9 +10,9 @@ import { TemplateAccumulator, collectCommitRevisionDetails, collectOverrideRevisionDetails, - collectCustomPlotsData, + collectCustomPlots, getCustomPlotId, - collectCustomCheckpointPlotData + collectCustomCheckpointPlots } from './collect' import { getRevisionFirstThreeColumns } from './util' import { CustomPlotsOrderValue, isCheckpointPlot } from './custom' @@ -167,7 +167,7 @@ export class PlotsModel extends ModelWithPersistence { public recreateCustomPlots(data?: ExperimentsOutput) { if (data) { - this.customCheckpointPlots = collectCustomCheckpointPlotData(data) + this.customCheckpointPlots = collectCustomCheckpointPlots(data) } const experiments = this.experiments.getExperiments() @@ -176,7 +176,7 @@ export class PlotsModel extends ModelWithPersistence { this.customPlots = undefined return } - const customPlots: CustomPlot[] = collectCustomPlotsData( + const customPlots: CustomPlot[] = collectCustomPlots( this.getCustomPlotsOrder(), this.customCheckpointPlots || {}, experiments @@ -207,7 +207,7 @@ export class PlotsModel extends ModelWithPersistence { public removeCustomPlots(plotIds: string[]) { const newCustomPlotsOrder = this.getCustomPlotsOrder().filter( - plot => !plotIds.includes(getCustomPlotId(plot)) + ({ metric, param }) => !plotIds.includes(getCustomPlotId(metric, param)) ) this.setCustomPlotsOrder(newCustomPlotsOrder) diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index 6e0b6dd415..ea293febf3 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -35,12 +35,13 @@ describe('pickCustomPlots', () => { it('should return the selected plots', async () => { const selectedPlots = [ - 'custom-metrics:summary.json:loss', + 'custom-metrics:summary.json:loss-epoch', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' ] const mockedPlots = [ { metric: 'metrics:summary.json:loss', + param: 'epoch', type: CustomPlotType.CHECKPOINT }, { @@ -70,7 +71,7 @@ describe('pickCustomPlots', () => { description: 'Checkpoint Trend Plot', detail: 'metrics:summary.json:loss', label: 'loss', - value: 'custom-metrics:summary.json:loss' + value: 'custom-metrics:summary.json:loss-epoch' }, { description: 'Metric Vs Param Plot', diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index ae0e51488e..eb4a33fb9f 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -22,11 +22,7 @@ const getMetricVsParamPlotItem = (metric: string, param: string) => { label: `${splitMetric[splitMetric.length - 1]} vs ${ splitParam[splitParam.length - 1] }`, - value: getCustomPlotId({ - metric, - param, - type: CustomPlotType.METRIC_VS_PARAM - }) + value: getCustomPlotId(metric, param) } } @@ -36,10 +32,7 @@ const getCheckpointPlotItem = (metric: string) => { description: 'Checkpoint Trend Plot', detail: metric, label: splitMetric[splitMetric.length - 1], - value: getCustomPlotId({ - metric, - type: CustomPlotType.CHECKPOINT - }) + value: getCustomPlotId(metric) } } @@ -53,7 +46,7 @@ export const pickCustomPlots = ( } const plotsItems = plots.map(plot => - isCheckpointValue(plot) + isCheckpointValue(plot.type) ? getCheckpointPlotItem(plot.metric) : getMetricVsParamPlotItem(plot.metric, plot.param) ) diff --git a/extension/src/plots/webview/contract.ts b/extension/src/plots/webview/contract.ts index 86cae482fa..ff4c1c2a41 100644 --- a/extension/src/plots/webview/contract.ts +++ b/extension/src/plots/webview/contract.ts @@ -69,7 +69,7 @@ export type MetricVsParamPlotValues = { expName: string metric: number param: number -} +}[] export type CheckpointPlotValues = { group: string @@ -81,25 +81,25 @@ export type ColorScale = { domain: string[]; range: Color[] } export type CheckpointPlot = { id: string - metric: string values: CheckpointPlotValues + metric: string + param: string type: CustomPlotType.CHECKPOINT } -export type CheckpointPlotData = CheckpointPlot & { yTitle: string } - export type MetricVsParamPlot = { id: string - values: MetricVsParamPlotValues[] + values: MetricVsParamPlotValues metric: string param: string type: CustomPlotType.METRIC_VS_PARAM } -export type MetricVsParamPlotData = MetricVsParamPlot & { yTitle: string } - export type CustomPlot = MetricVsParamPlot | CheckpointPlot -export type CustomPlotData = MetricVsParamPlotData | CheckpointPlotData + +export type CustomPlotData = CustomPlot & { + yTitle: string +} export type CustomPlotsData = { plots: CustomPlotData[] diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 09acde7015..5edf126140 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -32,7 +32,12 @@ import { Title } from '../../vscode/title' import { ColumnType } from '../../experiments/webview/contract' import { FILE_SEPARATOR } from '../../experiments/columns/paths' import { reorderObjectList } from '../../util/array' -import { CustomPlotsOrderValue, isCheckpointValue } from '../model/custom' +import { + CHECKPOINTS_PARAM, + CustomPlotsOrderValue, + doesCustomPlotAlreadyExist, + isCheckpointValue +} from '../model/custom' export class WebviewMessages { private readonly paths: PathsModel @@ -117,39 +122,6 @@ export class WebviewMessages { } } - public async addMetricVsParamPlot(): Promise< - CustomPlotsOrderValue | undefined - > { - const metricAndParam = await pickMetricAndParam( - this.experiments.getColumnTerminalNodes() - ) - - if (!metricAndParam) { - return - } - - const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { - if (isCheckpointValue(value)) { - return - } - return ( - value.param === metricAndParam.param && - value.metric === metricAndParam.metric - ) - }) - - if (plotAlreadyExists) { - return Toast.showError('Custom plot already exists.') - } - - const plot = { - ...metricAndParam, - type: CustomPlotType.METRIC_VS_PARAM - } - this.plots.addCustomPlot(plot) - this.sendCustomPlotsAndEvent(EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED) - } - private setPlotSize( section: Section, nbItemsPerRow: number, @@ -204,6 +176,35 @@ export class WebviewMessages { ) } + private async addMetricVsParamPlot(): Promise< + CustomPlotsOrderValue | undefined + > { + const metricAndParam = await pickMetricAndParam( + this.experiments.getColumnTerminalNodes() + ) + + if (!metricAndParam) { + return + } + + const plotAlreadyExists = doesCustomPlotAlreadyExist( + this.plots.getCustomPlotsOrder(), + metricAndParam.metric, + metricAndParam.param + ) + + if (plotAlreadyExists) { + return Toast.showError('Custom plot already exists.') + } + + const plot = { + ...metricAndParam, + type: CustomPlotType.METRIC_VS_PARAM + } + this.plots.addCustomPlot(plot) + this.sendCustomPlotsAndEvent(EventName.VIEWS_PLOTS_CUSTOM_PLOT_ADDED) + } + private async addCheckpointPlot(): Promise< CustomPlotsOrderValue | undefined > { @@ -213,11 +214,10 @@ export class WebviewMessages { return } - const plotAlreadyExists = this.plots.getCustomPlotsOrder().find(value => { - if (isCheckpointValue(value)) { - return value.metric === metric - } - }) + const plotAlreadyExists = doesCustomPlotAlreadyExist( + this.plots.getCustomPlotsOrder(), + metric + ) if (plotAlreadyExists) { return Toast.showError('Custom plot already exists.') @@ -225,6 +225,7 @@ export class WebviewMessages { const plot: CustomPlotsOrderValue = { metric, + param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT } this.plots.addCustomPlot(plot) @@ -243,7 +244,7 @@ export class WebviewMessages { return } - if (plotType === CustomPlotType.CHECKPOINT) { + if (isCheckpointValue(plotType)) { void this.addCheckpointPlot() } else { void this.addMetricVsParamPlot() @@ -275,22 +276,18 @@ export class WebviewMessages { const buildMetricOrParamPath = (type: string, path: string) => type + FILE_SEPARATOR + path + const newOrder: CustomPlotsOrderValue[] = reorderObjectList( plotIds, customPlots, 'id' - ).map(plot => - isCheckpointValue(plot) - ? { - metric: buildMetricOrParamPath(ColumnType.METRICS, plot.metric), - type: CustomPlotType.CHECKPOINT - } - : { - metric: buildMetricOrParamPath(ColumnType.METRICS, plot.metric), - param: buildMetricOrParamPath(ColumnType.PARAMS, plot.param), - type: CustomPlotType.METRIC_VS_PARAM - } - ) + ).map(plot => ({ + metric: buildMetricOrParamPath(ColumnType.METRICS, plot.metric), + param: isCheckpointValue(plot.type) + ? plot.param + : buildMetricOrParamPath(ColumnType.PARAMS, plot.param), + type: CustomPlotType.METRIC_VS_PARAM + })) this.plots.setCustomPlotsOrder(newOrder) this.sendCustomPlotsAndEvent(EventName.VIEWS_REORDER_PLOTS_CUSTOM) } diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index d04b98434e..d4b415eacf 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -1,5 +1,9 @@ import { copyOriginalColors } from '../../../../experiments/model/status/colors' -import { CustomPlotsOrderValue } from '../../../../plots/model/custom' +import { CustomCheckpointPlots } from '../../../../plots/model' +import { + CHECKPOINTS_PARAM, + CustomPlotsOrderValue +} from '../../../../plots/model/custom' import { CustomPlotsData, CustomPlotType, @@ -19,19 +23,22 @@ export const customPlotsOrderFixture: CustomPlotsOrderValue[] = [ }, { metric: 'metrics:summary.json:loss', + param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT }, { metric: 'metrics:summary.json:accuracy', + param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT } ] -export const checkpointPlotsFixture = { +export const checkpointPlotsFixture: CustomCheckpointPlots = { 'summary.json:loss': { - id: 'custom-summary.json:loss', + id: 'custom-summary.json:loss-epoch', metric: 'summary.json:loss', - type: 'checkpoint', + param: CHECKPOINTS_PARAM, + type: CustomPlotType.CHECKPOINT, values: [ { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, @@ -48,9 +55,10 @@ export const checkpointPlotsFixture = { ] }, 'summary.json:accuracy': { - id: 'custom-summary.json:accuracy', + id: 'custom-summary.json:accuracy-epoch', metric: 'summary.json:accuracy', - type: 'checkpoint', + param: CHECKPOINTS_PARAM, + type: CustomPlotType.CHECKPOINT, values: [ { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, @@ -67,9 +75,10 @@ export const checkpointPlotsFixture = { ] }, 'summary.json:val_loss': { - id: 'custom-summary.json:val_loss', + id: 'custom-summary.json:val_loss-epoch', metric: 'summary.json:val_loss', - type: 'checkpoint', + param: CHECKPOINTS_PARAM, + type: CustomPlotType.CHECKPOINT, values: [ { group: 'exp-83425', iteration: 1, y: 1.9391471147537231 }, { group: 'exp-83425', iteration: 2, y: 1.8825950622558594 }, @@ -86,9 +95,10 @@ export const checkpointPlotsFixture = { ] }, 'summary.json:val_accuracy': { - id: 'custom-summary.json:val_accuracy', + id: 'custom-summary.json:val_accuracy-epoch', metric: 'summary.json:val_accuracy', - type: 'checkpoint', + param: CHECKPOINTS_PARAM, + type: CustomPlotType.CHECKPOINT, values: [ { group: 'exp-83425', iteration: 1, y: 0.49399998784065247 }, { group: 'exp-83425', iteration: 2, y: 0.5550000071525574 }, @@ -163,8 +173,9 @@ const data: CustomPlotsData = { yTitle: 'summary.json:accuracy' }, { - id: 'custom-summary.json:loss', + id: 'custom-summary.json:loss-epoch', metric: 'summary.json:loss', + param: CHECKPOINTS_PARAM, values: [ { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, @@ -183,8 +194,9 @@ const data: CustomPlotsData = { yTitle: 'summary.json:loss' }, { - id: 'custom-summary.json:accuracy', + id: 'custom-summary.json:accuracy-epoch', metric: 'summary.json:accuracy', + param: CHECKPOINTS_PARAM, values: [ { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index a4ce2c0376..fa449f68bb 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -43,6 +43,7 @@ import { } from '../../../cli/dvc/contract' import { SelectedExperimentWithColor } from '../../../experiments/model' import * as customPlotQuickPickUtil from '../../../plots/model/quickPick' +import { CHECKPOINTS_PARAM } from '../../../plots/model/custom' suite('Plots Test Suite', () => { const disposable = Disposable.fn() @@ -825,6 +826,7 @@ suite('Plots Test Suite', () => { const mockCheckpointsOrderValue = { metric: 'metrics:summary.json:val_loss', + param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT } diff --git a/webview/src/plots/components/App.test.tsx b/webview/src/plots/components/App.test.tsx index 030388701d..7898659240 100644 --- a/webview/src/plots/components/App.test.tsx +++ b/webview/src/plots/components/App.test.tsx @@ -647,8 +647,8 @@ describe('App', () => { expect(plots.map(plot => plot.id)).toStrictEqual([ 'custom-metrics:summary.json:loss-params:params.yaml:dropout', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:loss', - 'custom-summary.json:accuracy' + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-epoch' ]) dragAndDrop(plots[1], plots[0]) @@ -658,8 +658,8 @@ describe('App', () => { expect(plots.map(plot => plot.id)).toStrictEqual([ 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-summary.json:loss', - 'custom-summary.json:accuracy' + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-epoch' ]) }) @@ -673,8 +673,8 @@ describe('App', () => { expect(plots.map(plot => plot.id)).toStrictEqual([ 'custom-metrics:summary.json:loss-params:params.yaml:dropout', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:loss', - 'custom-summary.json:accuracy' + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-epoch' ]) mockPostMessage.mockClear() @@ -682,10 +682,10 @@ describe('App', () => { dragAndDrop(plots[2], plots[0]) const expectedOrder = [ - 'custom-summary.json:loss', + 'custom-summary.json:loss-epoch', 'custom-metrics:summary.json:loss-params:params.yaml:dropout', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:accuracy' + 'custom-summary.json:accuracy-epoch' ] expect(mockPostMessage).toHaveBeenCalledTimes(1) @@ -712,7 +712,7 @@ describe('App', () => { ).toStrictEqual([ 'custom-metrics:summary.json:loss-params:params.yaml:dropout', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:loss' + 'custom-summary.json:loss-epoch' ]) sendSetDataMessage({ @@ -724,8 +724,8 @@ describe('App', () => { ).toStrictEqual([ 'custom-metrics:summary.json:loss-params:params.yaml:dropout', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:loss', - 'custom-summary.json:accuracy' + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-epoch' ]) }) @@ -740,8 +740,8 @@ describe('App', () => { ).toStrictEqual([ 'custom-metrics:summary.json:loss-params:params.yaml:dropout', 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:loss', - 'custom-summary.json:accuracy' + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-epoch' ]) sendSetDataMessage({ @@ -755,8 +755,8 @@ describe('App', () => { screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:loss', - 'custom-summary.json:accuracy' + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-epoch' ]) }) @@ -998,8 +998,8 @@ describe('App', () => { const expectedOrder = [ 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-summary.json:loss', - 'custom-summary.json:accuracy', + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-epoch', 'custom-metrics:summary.json:loss-params:params.yaml:dropout' ] @@ -1322,6 +1322,7 @@ describe('App', () => { plots.push({ id, metric: '', + param: '', type: CustomPlotType.CHECKPOINT, values: [], yTitle: id diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index 68e9826a7b..9378e44736 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -27,7 +27,7 @@ const createCustomPlotSpec = ( } if (isCheckpointPlot(plot)) { - return createCheckpointSpec(plot.yTitle, colors) + return createCheckpointSpec(plot.yTitle, plot.param, colors) } return createMetricVsParamSpec(plot.yTitle, plot.param) } diff --git a/webview/src/plots/components/customPlots/util.ts b/webview/src/plots/components/customPlots/util.ts index b945a15ecc..f81d3af57b 100644 --- a/webview/src/plots/components/customPlots/util.ts +++ b/webview/src/plots/components/customPlots/util.ts @@ -3,6 +3,7 @@ import { ColorScale } from 'dvc/src/plots/webview/contract' export const createCheckpointSpec = ( title: string, + param: string, scale?: ColorScale ): VisualizationSpec => ({ @@ -19,7 +20,7 @@ export const createCheckpointSpec = ( x: { axis: { format: '0d', tickMinStep: 1 }, field: 'iteration', - title: 'iteration', + title: param, type: 'quantitative' }, y: { diff --git a/webview/src/plots/components/plotDataStore.ts b/webview/src/plots/components/plotDataStore.ts index 9d62f88f9c..ca72a9039f 100644 --- a/webview/src/plots/components/plotDataStore.ts +++ b/webview/src/plots/components/plotDataStore.ts @@ -1,5 +1,4 @@ import { - CheckpointPlotData, CustomPlotData, Section, TemplatePlotEntry @@ -15,7 +14,7 @@ export const plotDataStore = { } export const addPlotsWithSnapshots = ( - plots: (CheckpointPlotData | TemplatePlotEntry | CustomPlotData)[], + plots: (TemplatePlotEntry | CustomPlotData)[], section: Section ) => { const snapShots: { [key: string]: string } = {} diff --git a/webview/src/plots/hooks/useGetPlot.ts b/webview/src/plots/hooks/useGetPlot.ts index aafa6e1ce8..82572b8654 100644 --- a/webview/src/plots/hooks/useGetPlot.ts +++ b/webview/src/plots/hooks/useGetPlot.ts @@ -1,5 +1,4 @@ import { - CheckpointPlotData, CustomPlotData, Section, TemplatePlotEntry @@ -33,7 +32,7 @@ export const useGetPlot = ( } if (isPlotWithSpec) { - setData({ values: (plot as CheckpointPlotData | CustomPlotData).values }) + setData({ values: (plot as CustomPlotData).values }) setContent(spec) return } From da9791c7f532607a85269e324f4838855527be36 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 10:46:17 -0500 Subject: [PATCH 24/40] Delete unneeded code --- extension/src/telemetry/constants.ts | 2 -- extension/src/webview/contract.ts | 10 ---------- 2 files changed, 12 deletions(-) diff --git a/extension/src/telemetry/constants.ts b/extension/src/telemetry/constants.ts index b00f375dfc..51da6ca6c2 100644 --- a/extension/src/telemetry/constants.ts +++ b/extension/src/telemetry/constants.ts @@ -76,7 +76,6 @@ export const EventName = Object.assign( VIEWS_PLOTS_SELECT_PLOTS: 'view.plots.selectPlots', VIEWS_PLOTS_ZOOM_PLOT: 'views.plots.zoomPlot', VIEWS_REORDER_PLOTS_CUSTOM: 'views.plots.customReordered', - VIEWS_REORDER_PLOTS_METRICS: 'views.plots.metricsReordered', VIEWS_REORDER_PLOTS_TEMPLATES: 'views.plots.templatesReordered', VIEWS_SETUP_CLOSE: 'view.setup.closed', @@ -263,7 +262,6 @@ export interface IEventNamePropertyMapping { [EventName.VIEWS_PLOTS_SELECT_PLOTS]: undefined [EventName.VIEWS_PLOTS_EXPERIMENT_TOGGLE]: undefined [EventName.VIEWS_PLOTS_ZOOM_PLOT]: undefined - [EventName.VIEWS_REORDER_PLOTS_METRICS]: undefined [EventName.VIEWS_REORDER_PLOTS_CUSTOM]: undefined [EventName.VIEWS_REORDER_PLOTS_TEMPLATES]: undefined diff --git a/extension/src/webview/contract.ts b/extension/src/webview/contract.ts index ec638fa710..9e27ed29df 100644 --- a/extension/src/webview/contract.ts +++ b/extension/src/webview/contract.ts @@ -29,7 +29,6 @@ export enum MessageFromWebviewType { REORDER_COLUMNS = 'reorder-columns', REORDER_PLOTS_COMPARISON = 'reorder-plots-comparison', REORDER_PLOTS_COMPARISON_ROWS = 'reorder-plots-comparison-rows', - REORDER_PLOTS_METRICS = 'reorder-plots-metrics', REORDER_PLOTS_CUSTOM = 'reorder-plots-custom', REORDER_PLOTS_TEMPLATES = 'reorder-plots-templates', REFRESH_REVISION = 'refresh-revision', @@ -53,7 +52,6 @@ export enum MessageFromWebviewType { SET_STUDIO_SHARE_EXPERIMENTS_LIVE = 'set-studio-share-experiments-live', SHARE_EXPERIMENT_AS_BRANCH = 'share-experiment-as-branch', SHARE_EXPERIMENT_AS_COMMIT = 'share-experiment-as-commit', - TOGGLE_METRIC = 'toggle-metric', TOGGLE_PLOTS_SECTION = 'toggle-plots-section', REMOVE_CUSTOM_PLOTS = 'remove-custom-plots', REMOVE_STUDIO_TOKEN = 'remove-studio-token', @@ -160,10 +158,6 @@ export type MessageFromWebview = type: MessageFromWebviewType.REMOVE_COLUMN_SORT payload: string } - | { - type: MessageFromWebviewType.TOGGLE_METRIC - payload: string[] - } | { type: MessageFromWebviewType.REMOVE_CUSTOM_PLOTS } @@ -176,10 +170,6 @@ export type MessageFromWebview = type: MessageFromWebviewType.REORDER_PLOTS_COMPARISON_ROWS payload: string[] } - | { - type: MessageFromWebviewType.REORDER_PLOTS_METRICS - payload: string[] - } | { type: MessageFromWebviewType.REORDER_PLOTS_CUSTOM payload: string[] From ace4b171d4d80360b59743ea5e5c2b5982849c5e Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 12:32:16 -0500 Subject: [PATCH 25/40] Resolve most review comments --- extension/resources/walkthrough/live-plots.md | 6 +- extension/resources/walkthrough/plots.md | 2 +- extension/src/plots/model/collect.test.ts | 82 ++++++++++++++++--- extension/src/plots/model/collect.ts | 29 ++++++- extension/src/plots/model/index.ts | 29 +++---- extension/src/plots/webview/messages.ts | 11 +-- .../customPlots/customPlotsSlice.ts | 6 +- webview/src/plots/hooks/useGetPlot.ts | 11 +-- 8 files changed, 126 insertions(+), 50 deletions(-) diff --git a/extension/resources/walkthrough/live-plots.md b/extension/resources/walkthrough/live-plots.md index da2d3605bc..e945400abd 100644 --- a/extension/resources/walkthrough/live-plots.md +++ b/extension/resources/walkthrough/live-plots.md @@ -33,6 +33,6 @@ for epoch in range(NUM_EPOCHS): `DVCLive` is _optional_, and you can just append or modify plot files using any language and any tool. -💡 `Custom` section of the plots dashboard is being updated automatically based -on the data in the table. You don't even have to manage or write any special -plot files. +💡 Plots created in the `Custom` section of the plots dashboard is being updated +automatically based on the data in the table. You don't even have to manage or +write any special plot files. diff --git a/extension/resources/walkthrough/plots.md b/extension/resources/walkthrough/plots.md index 37ff662135..eab44a5b09 100644 --- a/extension/resources/walkthrough/plots.md +++ b/extension/resources/walkthrough/plots.md @@ -75,7 +75,7 @@ rendered side by side for the selected experiments.

**Custom** plots are generated linear plots comparing metrics and params. A user -can choose between two types of plots, "Checkpoint Trend" and "Metric Vs Param". +can add two types of plots, "Checkpoint Trend" and "Metric Vs Param". "Metric Vs Param" plots compare a chosen metric and param across experiments. "Checkpoint Trend" plots can compare a chosen [metric] value per epoch if diff --git a/extension/src/plots/model/collect.test.ts b/extension/src/plots/model/collect.test.ts index 1b08078f07..11330c4b09 100644 --- a/extension/src/plots/model/collect.test.ts +++ b/extension/src/plots/model/collect.test.ts @@ -6,7 +6,8 @@ import { collectTemplates, collectOverrideRevisionDetails, collectCustomPlots, - collectCustomCheckpointPlots + collectCustomCheckpointPlots, + collectCustomPlotData } from './collect' import plotsDiffFixture from '../../test/fixtures/plotsDiff/output' import customPlotsFixture, { @@ -18,7 +19,12 @@ import { EXPERIMENT_WORKSPACE_ID } from '../../cli/dvc/contract' import { sameContents } from '../../util/array' -import { CustomPlot, TemplatePlot } from '../webview/contract' +import { + CheckpointPlot, + CustomPlot, + CustomPlotData, + TemplatePlot +} from '../webview/contract' import { getCLICommitId } from '../../test/fixtures/plotsDiff/util' import expShowFixture from '../../test/fixtures/expShow/base/output' import modifiedFixture from '../../test/fixtures/expShow/modified/output' @@ -29,17 +35,25 @@ const logsLossPath = join('logs', 'loss.tsv') const logsLossPlot = (plotsDiffFixture[logsLossPath][0] || {}) as TemplatePlot +const getCustomPlotFromCustomPlotData = ({ + id, + metric, + param, + type, + values +}: CustomPlotData) => + ({ + id, + metric, + param, + type, + values + } as CustomPlot) + describe('collectCustomPlots', () => { it('should return the expected data from the test fixture', () => { const expectedOutput: CustomPlot[] = customPlotsFixture.plots.map( - ({ type, metric, id, values, param }) => - ({ - id, - metric, - param, - type, - values - } as CustomPlot) + getCustomPlotFromCustomPlotData ) const data = collectCustomPlots( customPlotsOrderFixture, @@ -88,6 +102,54 @@ describe('collectCustomPlots', () => { }) }) +describe('collectCustomPlotData', () => { + it('should return the expected data from test fixture', () => { + const expectedMetricVsParamPlotData = customPlotsFixture.plots[0] + const expectedCheckpointsPlotData = customPlotsFixture.plots[2] + const metricVsParamPlot = getCustomPlotFromCustomPlotData( + expectedMetricVsParamPlotData + ) + const checkpointsPlot = getCustomPlotFromCustomPlotData( + expectedCheckpointsPlotData + ) + + const metricVsParamData = collectCustomPlotData( + metricVsParamPlot, + customPlotsFixture.colors, + customPlotsFixture.nbItemsPerRow + ) + + const checkpointsData = collectCustomPlotData( + { + ...checkpointsPlot, + values: [ + ...checkpointsPlot.values, + { + group: 'exp-123', + iteration: 1, + y: 1.4534177053451538 + }, + { + group: 'exp-123', + iteration: 2, + y: 1.757687 + }, + { + group: 'exp-123', + iteration: 3, + y: 1.989894 + } + ] + } as CheckpointPlot, + customPlotsFixture.colors, + customPlotsFixture.nbItemsPerRow + ) + + expect(metricVsParamData).toStrictEqual(expectedMetricVsParamPlotData) + expect(checkpointsData).toStrictEqual(expectedCheckpointsPlotData) + }) +}) + describe('collectData', () => { it('should return the expected output from the test fixture', () => { const mapping = { diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index b1a3dae62e..7e56fb004b 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -6,6 +6,7 @@ import { CustomCheckpointPlots } from '.' import { CHECKPOINTS_PARAM, CustomPlotsOrderValue, + isCheckpointPlot, isCheckpointValue } from './custom' import { getRevisionFirstThreeColumns } from './util' @@ -22,7 +23,8 @@ import { Revision, CustomPlotType, CustomPlot, - MetricVsParamPlot + MetricVsParamPlot, + CustomPlotData } from '../webview/contract' import { EXPERIMENT_WORKSPACE_ID, @@ -48,7 +50,11 @@ import { } from '../../experiments/webview/contract' import { addToMapArray } from '../../util/map' import { TemplateOrder } from '../paths/collect' -import { extendVegaSpec, isMultiViewPlot } from '../vega/util' +import { + extendVegaSpec, + isMultiViewPlot, + truncateVerticalTitle +} from '../vega/util' import { definedAndNonEmpty, reorderObjectList } from '../../util/array' import { shortenForLabel } from '../../util/string' import { @@ -307,6 +313,25 @@ export const collectCustomPlots = ( .filter(Boolean) } +export const collectCustomPlotData = ( + plot: CustomPlot, + colors: ColorScale | undefined, + nbItemsPerRow: number +): CustomPlotData => { + const selectedExperiments = colors?.domain + const filteredValues = isCheckpointPlot(plot) + ? plot.values.filter(value => + (selectedExperiments as string[]).includes(value.group) + ) + : plot.values + + return { + ...plot, + values: filteredValues, + yTitle: truncateVerticalTitle(plot.metric, nbItemsPerRow) as string + } as CustomPlotData +} + type RevisionPathData = { [path: string]: Record[] } export type RevisionData = { diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 6fb137b79e..d5489e19a7 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -12,7 +12,8 @@ import { collectOverrideRevisionDetails, collectCustomPlots, getCustomPlotId, - collectCustomCheckpointPlots + collectCustomCheckpointPlots, + collectCustomPlotData } from './collect' import { getRevisionFirstThreeColumns } from './util' import { CustomPlotsOrderValue, isCheckpointPlot } from './custom' @@ -39,7 +40,7 @@ import { PlotsOutputOrError } from '../../cli/dvc/contract' import { Experiments } from '../../experiments' -import { getColorScale, truncateVerticalTitle } from '../vega/util' +import { getColorScale } from '../vega/util' import { definedAndNonEmpty, reorderObjectList } from '../../util/array' import { removeMissingKeysFromObject } from '../../util/object' import { TemplateOrder } from '../paths/collect' @@ -161,7 +162,7 @@ export class PlotsModel extends ModelWithPersistence { colors, height: this.getHeight(Section.CUSTOM_PLOTS), nbItemsPerRow: this.getNbItemsPerRow(Section.CUSTOM_PLOTS), - plots: this.getCustomPlotData(this.customPlots, colors) + plots: this.getCustomPlotsData(this.customPlots, colors) } } @@ -457,7 +458,7 @@ export class PlotsModel extends ModelWithPersistence { return this.commitRevisions[label] || label } - private getCustomPlotData( + private getCustomPlotsData( plots: CustomPlot[], colors: ColorScale | undefined ): CustomPlotData[] { @@ -466,20 +467,12 @@ export class PlotsModel extends ModelWithPersistence { if (!selectedExperimentsExist) { filteredPlots = plots.filter(plot => !isCheckpointPlot(plot)) } - return filteredPlots.map( - plot => - ({ - ...plot, - values: isCheckpointPlot(plot) - ? plot.values.filter(value => - (colors as ColorScale).domain.includes(value.group) - ) - : plot.values, - yTitle: truncateVerticalTitle( - plot.metric, - this.getNbItemsPerRow(Section.CUSTOM_PLOTS) - ) as string - } as CustomPlotData) + return filteredPlots.map(plot => + collectCustomPlotData( + plot, + colors, + this.getNbItemsPerRow(Section.CUSTOM_PLOTS) + ) ) } diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 5edf126140..0bd5490478 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -234,8 +234,7 @@ export class WebviewMessages { private async addCustomPlot() { if (!this.experiments.hasCheckpoints()) { - void this.addMetricVsParamPlot() - return + return this.addMetricVsParamPlot() } const plotType = await pickCustomPlotType() @@ -244,11 +243,9 @@ export class WebviewMessages { return } - if (isCheckpointValue(plotType)) { - void this.addCheckpointPlot() - } else { - void this.addMetricVsParamPlot() - } + return isCheckpointValue(plotType) + ? this.addCheckpointPlot() + : this.addMetricVsParamPlot() } private async removeCustomPlots() { diff --git a/webview/src/plots/components/customPlots/customPlotsSlice.ts b/webview/src/plots/components/customPlots/customPlotsSlice.ts index 5e2cb2da38..db060ed7ee 100644 --- a/webview/src/plots/components/customPlots/customPlotsSlice.ts +++ b/webview/src/plots/components/customPlots/customPlotsSlice.ts @@ -16,8 +16,10 @@ export interface CustomPlotsState extends Omit { disabledDragPlotIds: string[] } +const initialColorsState = { domain: [], range: [] } + export const customPlotsInitialState: CustomPlotsState = { - colors: { domain: [], range: [] }, + colors: initialColorsState, disabledDragPlotIds: [], hasData: false, height: DEFAULT_HEIGHT[Section.CUSTOM_PLOTS], @@ -51,7 +53,7 @@ export const customPlotsSlice = createSlice({ return { ...state, ...statePayload, - colors: colors || { domain: [], range: [] }, + colors: colors || initialColorsState, hasData: !!action.payload, plotsIds: plots?.map(plot => plot.id) || [], plotsSnapshots: snapShots diff --git a/webview/src/plots/hooks/useGetPlot.ts b/webview/src/plots/hooks/useGetPlot.ts index 82572b8654..d1528d36c7 100644 --- a/webview/src/plots/hooks/useGetPlot.ts +++ b/webview/src/plots/hooks/useGetPlot.ts @@ -9,16 +9,13 @@ import { PlainObject, VisualizationSpec } from 'react-vega' import { plotDataStore } from '../components/plotDataStore' import { PlotsState } from '../store' -const getStoreSection = (section: Section) => - section === Section.TEMPLATE_PLOTS ? 'template' : 'custom' - export const useGetPlot = ( section: Section, id: string, spec?: VisualizationSpec ) => { - const isPlotWithSpec = section === Section.CUSTOM_PLOTS - const storeSection = getStoreSection(section) + const isCustomPlot = section === Section.CUSTOM_PLOTS + const storeSection = isCustomPlot ? 'custom' : 'template' const snapshot = useSelector( (state: PlotsState) => state[storeSection].plotsSnapshots ) @@ -31,7 +28,7 @@ export const useGetPlot = ( return } - if (isPlotWithSpec) { + if (isCustomPlot) { setData({ values: (plot as CustomPlotData).values }) setContent(spec) return @@ -43,7 +40,7 @@ export const useGetPlot = ( height: 'container', width: 'container' } as VisualizationSpec) - }, [id, isPlotWithSpec, setData, setContent, section, spec]) + }, [id, isCustomPlot, setData, setContent, section, spec]) useEffect(() => { setPlotData() From bf5bea89c28b2dbb5229a1bc4e42a08846ccb76a Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 17:52:44 -0500 Subject: [PATCH 26/40] Use only one loop with `getCustomPlotsData` --- extension/src/plots/model/index.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index d5489e19a7..216227a342 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -463,17 +463,20 @@ export class PlotsModel extends ModelWithPersistence { colors: ColorScale | undefined ): CustomPlotData[] { const selectedExperimentsExist = !!colors - let filteredPlots = plots - if (!selectedExperimentsExist) { - filteredPlots = plots.filter(plot => !isCheckpointPlot(plot)) - } - return filteredPlots.map(plot => - collectCustomPlotData( - plot, - colors, - this.getNbItemsPerRow(Section.CUSTOM_PLOTS) + const filteredPlots: CustomPlotData[] = [] + for (const plot of plots) { + if (!selectedExperimentsExist && isCheckpointPlot(plot)) { + continue + } + filteredPlots.push( + collectCustomPlotData( + plot, + colors, + this.getNbItemsPerRow(Section.CUSTOM_PLOTS) + ) ) - ) + } + return filteredPlots } private getSelectedComparisonPlots( From 7b0790a0a7b8c171b05e1f34b7752c2babf3c1c1 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 18:08:36 -0500 Subject: [PATCH 27/40] Fix typo --- extension/src/plots/webview/messages.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 0bd5490478..b3b55c2c57 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -283,7 +283,7 @@ export class WebviewMessages { param: isCheckpointValue(plot.type) ? plot.param : buildMetricOrParamPath(ColumnType.PARAMS, plot.param), - type: CustomPlotType.METRIC_VS_PARAM + type: plot.type })) this.plots.setCustomPlotsOrder(newOrder) this.sendCustomPlotsAndEvent(EventName.VIEWS_REORDER_PLOTS_CUSTOM) From 5bfa20f3f22e34167c780ef8e2599c9a1929484c Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 18:17:02 -0500 Subject: [PATCH 28/40] Delete duplication code --- .../src/plots/components/PlotsContainer.tsx | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/webview/src/plots/components/PlotsContainer.tsx b/webview/src/plots/components/PlotsContainer.tsx index ae5e9f0b37..f3e551c0d6 100644 --- a/webview/src/plots/components/PlotsContainer.tsx +++ b/webview/src/plots/components/PlotsContainer.tsx @@ -31,43 +31,6 @@ export interface PlotsContainerProps { hasItems?: boolean } -export const SectionDescription = { - // "Custom" - [Section.CUSTOM_PLOTS]: ( - - Generated custom linear plots comparing chosen metrics and params in all - experiments in the table. - - ), - // "Images" - [Section.COMPARISON_TABLE]: ( - - Images (e.g. any .jpg, .svg, or - .png file) rendered side by side across experiments. They - should be registered as{' '} - - plots - - . - - ), - // "Data Series" - [Section.TEMPLATE_PLOTS]: ( - - Any JSON, YAML, CSV, or{' '} - TSV file(s) with data points, visualized using{' '} - - plot templates - - . Either predefined (e.g. confusion matrix, linear) or{' '} - - custom Vega-lite templates - - . - - ) -} - export const PlotsContainer: React.FC = ({ sectionCollapsed, sectionKey, From 244076a5d6040036da06ef3fbbd9d5835cc5b5d7 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 13 Mar 2023 19:19:07 -0500 Subject: [PATCH 29/40] Rename argument --- extension/src/plots/model/quickPick.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index eb4a33fb9f..2d0486b2c2 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -37,18 +37,18 @@ const getCheckpointPlotItem = (metric: string) => { } export const pickCustomPlots = ( - plots: CustomPlotsOrderValue[], + plotsOrderValues: CustomPlotsOrderValue[], noPlotsErrorMessage: string, quickPickOptions: QuickPickOptionsWithTitle ): Thenable => { - if (!definedAndNonEmpty(plots)) { + if (!definedAndNonEmpty(plotsOrderValues)) { return Toast.showError(noPlotsErrorMessage) } - const plotsItems = plots.map(plot => - isCheckpointValue(plot.type) - ? getCheckpointPlotItem(plot.metric) - : getMetricVsParamPlotItem(plot.metric, plot.param) + const plotsItems = plotsOrderValues.map(value => + isCheckpointValue(value.type) + ? getCheckpointPlotItem(value.metric) + : getMetricVsParamPlotItem(value.metric, value.param) ) return quickPickManyValues(plotsItems, quickPickOptions) From e46167759005da73844d6d40600f7562e2979541 Mon Sep 17 00:00:00 2001 From: Julie G <43496356+julieg18@users.noreply.github.com> Date: Mon, 13 Mar 2023 21:12:15 -0500 Subject: [PATCH 30/40] Clean up column path logic in custom state (#3460) --- extension/src/plots/model/collect.ts | 18 ++++++---- extension/src/plots/model/custom.ts | 21 ++++++++++++ extension/src/plots/model/index.ts | 19 +++++------ extension/src/plots/webview/messages.ts | 15 +++----- .../test/fixtures/expShow/base/customPlots.ts | 16 ++++----- extension/src/test/suite/plots/index.test.ts | 24 ++++++------- webview/src/plots/components/App.test.tsx | 34 +++++++++---------- 7 files changed, 83 insertions(+), 64 deletions(-) diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 7e56fb004b..8e23831469 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -40,7 +40,8 @@ import { extractColumns } from '../../experiments/columns/extract' import { decodeColumn, appendColumnToPath, - splitColumnPath + splitColumnPath, + FILE_SEPARATOR } from '../../experiments/columns/paths' import { ColumnType, @@ -270,18 +271,23 @@ const collectMetricVsParamPlot = ( param: string, experiments: Experiment[] ): MetricVsParamPlot => { - const splitUpMetricPath = splitColumnPath(metric) - const splitUpParamPath = splitColumnPath(param) + const splitUpMetricPath = splitColumnPath( + ColumnType.METRICS + FILE_SEPARATOR + metric + ) + const splitUpParamPath = splitColumnPath( + ColumnType.PARAMS + FILE_SEPARATOR + param + ) const plotData: MetricVsParamPlot = { id: getCustomPlotId(metric, param), - metric: metric.slice(ColumnType.METRICS.length + 1), - param: param.slice(ColumnType.PARAMS.length + 1), + metric, + param, type: CustomPlotType.METRIC_VS_PARAM, values: [] } for (const experiment of experiments) { const metricValue = get(experiment, splitUpMetricPath) as number | undefined + const paramValue = get(experiment, splitUpParamPath) as number | undefined if (metricValue !== undefined && paramValue !== undefined) { @@ -305,7 +311,7 @@ export const collectCustomPlots = ( .map((plotOrderValue): CustomPlot => { if (isCheckpointValue(plotOrderValue.type)) { const { metric } = plotOrderValue - return checkpointPlots[metric.slice(ColumnType.METRICS.length + 1)] + return checkpointPlots[metric] } const { metric, param } = plotOrderValue return collectMetricVsParamPlot(metric, param, experiments) diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts index 410198456b..5bd051ff52 100644 --- a/extension/src/plots/model/custom.ts +++ b/extension/src/plots/model/custom.ts @@ -1,3 +1,4 @@ +import { ColumnType } from '../../experiments/webview/contract' import { CheckpointPlot, CustomPlot, CustomPlotType } from '../webview/contract' export const CHECKPOINTS_PARAM = 'epoch' @@ -23,3 +24,23 @@ export const doesCustomPlotAlreadyExist = ( order.some(value => { return value.param === param && value.metric === metric }) + +const removeColumnTypeFromPath = ( + columnPath: string, + type: string, + fileSep: string +) => + columnPath.startsWith(type + fileSep) + ? columnPath.slice(type.length + 1) + : columnPath + +export const cleanupOldOrderValue = ( + { param, metric, type }: CustomPlotsOrderValue, + fileSep: string +): CustomPlotsOrderValue => ({ + // previous column paths have the "TYPE:" prefix + metric: removeColumnTypeFromPath(metric, ColumnType.METRICS, fileSep), + param: removeColumnTypeFromPath(param, ColumnType.PARAMS, fileSep), + // previous values didn't have a type + type: type || CustomPlotType.METRIC_VS_PARAM +}) diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index 216227a342..6d474229ee 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -16,7 +16,11 @@ import { collectCustomPlotData } from './collect' import { getRevisionFirstThreeColumns } from './util' -import { CustomPlotsOrderValue, isCheckpointPlot } from './custom' +import { + cleanupOldOrderValue, + CustomPlotsOrderValue, + isCheckpointPlot +} from './custom' import { CheckpointPlot, ComparisonPlots, @@ -31,8 +35,7 @@ import { CustomPlot, ColorScale, DEFAULT_HEIGHT, - DEFAULT_NB_ITEMS_PER_ROW, - CustomPlotType + DEFAULT_NB_ITEMS_PER_ROW } from '../webview/contract' import { ExperimentsOutput, @@ -53,6 +56,7 @@ import { MultiSourceVariations } from '../multiSource/collect' import { isDvcError } from '../../cli/dvc/reader' +import { FILE_SEPARATOR } from '../../experiments/columns/paths' export type CustomCheckpointPlots = { [metric: string]: CheckpointPlot } @@ -186,13 +190,8 @@ export class PlotsModel extends ModelWithPersistence { } public getCustomPlotsOrder() { - return this.customPlotsOrder.map( - ({ type, ...rest }) => - ({ - ...rest, - // type is possibly undefined if state holds an older version of custom plots - type: type || CustomPlotType.METRIC_VS_PARAM - } as CustomPlotsOrderValue) + return this.customPlotsOrder.map(value => + cleanupOldOrderValue(value, FILE_SEPARATOR) ) } diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index b3b55c2c57..a4664b2b9e 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -29,8 +29,6 @@ import { pickMetricAndParam } from '../model/quickPick' import { Title } from '../../vscode/title' -import { ColumnType } from '../../experiments/webview/contract' -import { FILE_SEPARATOR } from '../../experiments/columns/paths' import { reorderObjectList } from '../../util/array' import { CHECKPOINTS_PARAM, @@ -271,19 +269,14 @@ export class WebviewMessages { return } - const buildMetricOrParamPath = (type: string, path: string) => - type + FILE_SEPARATOR + path - const newOrder: CustomPlotsOrderValue[] = reorderObjectList( plotIds, customPlots, 'id' - ).map(plot => ({ - metric: buildMetricOrParamPath(ColumnType.METRICS, plot.metric), - param: isCheckpointValue(plot.type) - ? plot.param - : buildMetricOrParamPath(ColumnType.PARAMS, plot.param), - type: plot.type + ).map(({ metric, param, type }) => ({ + metric, + param, + type })) this.plots.setCustomPlotsOrder(newOrder) this.sendCustomPlotsAndEvent(EventName.VIEWS_REORDER_PLOTS_CUSTOM) diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index d4b415eacf..9e40d4db4f 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -12,22 +12,22 @@ import { export const customPlotsOrderFixture: CustomPlotsOrderValue[] = [ { - metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout', + metric: 'summary.json:loss', + param: 'params.yaml:dropout', type: CustomPlotType.METRIC_VS_PARAM }, { - metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:epochs', + metric: 'summary.json:accuracy', + param: 'params.yaml:epochs', type: CustomPlotType.METRIC_VS_PARAM }, { - metric: 'metrics:summary.json:loss', + metric: 'summary.json:loss', param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT }, { - metric: 'metrics:summary.json:accuracy', + metric: 'summary.json:accuracy', param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT } @@ -125,7 +125,7 @@ const data: CustomPlotsData = { }, plots: [ { - id: 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + id: 'custom-summary.json:loss-params.yaml:dropout', metric: 'summary.json:loss', param: 'params.yaml:dropout', type: CustomPlotType.METRIC_VS_PARAM, @@ -149,7 +149,7 @@ const data: CustomPlotsData = { yTitle: 'summary.json:loss' }, { - id: 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + id: 'custom-summary.json:accuracy-params.yaml:epochs', metric: 'summary.json:accuracy', param: 'params.yaml:epochs', type: CustomPlotType.METRIC_VS_PARAM, diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index fa449f68bb..bb45c6033d 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -431,8 +431,8 @@ suite('Plots Test Suite', () => { const webview = await plots.showWebview() const mockNewCustomPlotsOrder = [ - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-metrics:summary.json:loss-params:params.yaml:dropout' + 'custom-summary.json:accuracy-params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout' ] stub(plotsModel, 'getCustomPlots') @@ -454,13 +454,13 @@ suite('Plots Test Suite', () => { expect(mockSetCustomPlotsOrder).to.be.calledOnce expect(mockSetCustomPlotsOrder).to.be.calledWithExactly([ { - metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:epochs', + metric: 'summary.json:accuracy', + param: 'params.yaml:epochs', type: CustomPlotType.METRIC_VS_PARAM }, { - metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout', + metric: 'summary.json:loss', + param: 'params.yaml:dropout', type: CustomPlotType.METRIC_VS_PARAM } ]) @@ -781,8 +781,8 @@ suite('Plots Test Suite', () => { const mockGetMetric = stub(customPlotQuickPickUtil, 'pickMetric') const mockMetricVsParamOrderValue = { - metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:dropout', + metric: 'summary.json:accuracy', + param: 'params.yaml:dropout', type: CustomPlotType.METRIC_VS_PARAM } @@ -825,7 +825,7 @@ suite('Plots Test Suite', () => { ) const mockCheckpointsOrderValue = { - metric: 'metrics:summary.json:val_loss', + metric: 'summary.json:val_loss', param: CHECKPOINTS_PARAM, type: CustomPlotType.CHECKPOINT } @@ -877,15 +877,15 @@ suite('Plots Test Suite', () => { mockSelectCustomPlots.callsFake(() => { resolve(undefined) return Promise.resolve([ - 'custom-metrics:summary.json:loss-params:params.yaml:dropout' + 'custom-summary.json:loss-params.yaml:dropout' ]) }) ) stub(plotsModel, 'getCustomPlotsOrder').returns([ { - metric: 'metrics:summary.json:loss', - param: 'params:params.yaml:dropout', + metric: 'summary.json:loss', + param: 'params.yaml:dropout', type: CustomPlotType.METRIC_VS_PARAM } ]) diff --git a/webview/src/plots/components/App.test.tsx b/webview/src/plots/components/App.test.tsx index 7898659240..a8494b1052 100644 --- a/webview/src/plots/components/App.test.tsx +++ b/webview/src/plots/components/App.test.tsx @@ -645,8 +645,8 @@ describe('App', () => { let plots = screen.getAllByTestId(/summary\.json/) expect(plots.map(plot => plot.id)).toStrictEqual([ - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch', 'custom-summary.json:accuracy-epoch' ]) @@ -656,8 +656,8 @@ describe('App', () => { plots = screen.getAllByTestId(/summary\.json/) expect(plots.map(plot => plot.id)).toStrictEqual([ - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', + 'custom-summary.json:accuracy-params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout', 'custom-summary.json:loss-epoch', 'custom-summary.json:accuracy-epoch' ]) @@ -671,8 +671,8 @@ describe('App', () => { const plots = screen.getAllByTestId(/summary\.json/) expect(plots.map(plot => plot.id)).toStrictEqual([ - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch', 'custom-summary.json:accuracy-epoch' ]) @@ -683,8 +683,8 @@ describe('App', () => { const expectedOrder = [ 'custom-summary.json:loss-epoch', - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:accuracy-epoch' ] @@ -710,8 +710,8 @@ describe('App', () => { expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch' ]) @@ -722,8 +722,8 @@ describe('App', () => { expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch', 'custom-summary.json:accuracy-epoch' ]) @@ -738,8 +738,8 @@ describe('App', () => { expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ - 'custom-metrics:summary.json:loss-params:params.yaml:dropout', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:loss-params.yaml:dropout', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch', 'custom-summary.json:accuracy-epoch' ]) @@ -754,7 +754,7 @@ describe('App', () => { expect( screen.getAllByTestId(/summary\.json/).map(plot => plot.id) ).toStrictEqual([ - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch', 'custom-summary.json:accuracy-epoch' ]) @@ -997,10 +997,10 @@ describe('App', () => { dragAndDrop(plots[0], screen.getByTestId('custom-plots')) const expectedOrder = [ - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs', + 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch', 'custom-summary.json:accuracy-epoch', - 'custom-metrics:summary.json:loss-params:params.yaml:dropout' + 'custom-summary.json:loss-params.yaml:dropout' ] expect( From 38ab2d678d748721de4243e15a7d8792e00c6099 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Tue, 14 Mar 2023 09:38:06 -0500 Subject: [PATCH 31/40] Fix quick pick --- extension/src/plots/model/collect.ts | 6 ++-- extension/src/plots/model/custom.ts | 6 ++++ extension/src/plots/model/quickPick.test.ts | 25 ++++++++-------- extension/src/plots/model/quickPick.ts | 33 ++++++++++++++++----- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 8e23831469..02360cdb07 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -4,6 +4,7 @@ import { TopLevelSpec } from 'vega-lite' import { VisualizationSpec } from 'react-vega' import { CustomCheckpointPlots } from '.' import { + getFullValuePath, CHECKPOINTS_PARAM, CustomPlotsOrderValue, isCheckpointPlot, @@ -272,10 +273,10 @@ const collectMetricVsParamPlot = ( experiments: Experiment[] ): MetricVsParamPlot => { const splitUpMetricPath = splitColumnPath( - ColumnType.METRICS + FILE_SEPARATOR + metric + getFullValuePath(ColumnType.METRICS, metric, FILE_SEPARATOR) ) const splitUpParamPath = splitColumnPath( - ColumnType.PARAMS + FILE_SEPARATOR + param + getFullValuePath(ColumnType.PARAMS, param, FILE_SEPARATOR) ) const plotData: MetricVsParamPlot = { id: getCustomPlotId(metric, param), @@ -287,7 +288,6 @@ const collectMetricVsParamPlot = ( for (const experiment of experiments) { const metricValue = get(experiment, splitUpMetricPath) as number | undefined - const paramValue = get(experiment, splitUpParamPath) as number | undefined if (metricValue !== undefined && paramValue !== undefined) { diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts index 5bd051ff52..3f6fcb0a6e 100644 --- a/extension/src/plots/model/custom.ts +++ b/extension/src/plots/model/custom.ts @@ -34,6 +34,12 @@ const removeColumnTypeFromPath = ( ? columnPath.slice(type.length + 1) : columnPath +export const getFullValuePath = ( + type: string, + columnPath: string, + fileSep: string +) => type + fileSep + columnPath + export const cleanupOldOrderValue = ( { param, metric, type }: CustomPlotsOrderValue, fileSep: string diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index ea293febf3..20f9fbf5a9 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -35,23 +35,23 @@ describe('pickCustomPlots', () => { it('should return the selected plots', async () => { const selectedPlots = [ - 'custom-metrics:summary.json:loss-epoch', - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' + 'custom-summary.json:loss-epoch', + 'custom-summary.json:accuracy-params.yaml:epochs' ] const mockedPlots = [ { - metric: 'metrics:summary.json:loss', + metric: 'summary.json:loss', param: 'epoch', type: CustomPlotType.CHECKPOINT }, { - metric: 'metrics:summary.json:accuracy', - param: 'params:params.yaml:epochs', + metric: 'summary.json:accuracy', + param: 'params.yaml:epochs', type: CustomPlotType.METRIC_VS_PARAM }, { - metric: 'metrics:summary.json:learning_rate', - param: 'param:summary.json:process.threshold', + metric: 'summary.json:learning_rate', + param: 'summary.json:process.threshold', type: CustomPlotType.METRIC_VS_PARAM } ] as CustomPlotsOrderValue[] @@ -71,22 +71,21 @@ describe('pickCustomPlots', () => { description: 'Checkpoint Trend Plot', detail: 'metrics:summary.json:loss', label: 'loss', - value: 'custom-metrics:summary.json:loss-epoch' + value: 'custom-summary.json:loss-epoch' }, { description: 'Metric Vs Param Plot', detail: 'metrics:summary.json:accuracy vs params:params.yaml:epochs', label: 'accuracy vs epochs', - value: - 'custom-metrics:summary.json:accuracy-params:params.yaml:epochs' + value: 'custom-summary.json:accuracy-params.yaml:epochs' }, { description: 'Metric Vs Param Plot', detail: - 'metrics:summary.json:learning_rate vs param:summary.json:process.threshold', + 'metrics:summary.json:learning_rate vs params:summary.json:process.threshold', label: 'learning_rate vs threshold', value: - 'custom-metrics:summary.json:learning_rate-param:summary.json:process.threshold' + 'custom-summary.json:learning_rate-summary.json:process.threshold' } ], { title: Title.SELECT_CUSTOM_PLOTS_TO_REMOVE } @@ -184,7 +183,7 @@ describe('pickMetricAndParam', () => { } const expectedParam = { label: 'epochs', - path: 'summary.json:loss-params:params.yaml:epochs' + path: 'summary.json:loss-params.yaml:epochs' } mockedQuickPickValue .mockResolvedValueOnce(expectedMetric) diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index 2d0486b2c2..fe491679ed 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -1,6 +1,13 @@ import { getCustomPlotId } from './collect' -import { CustomPlotsOrderValue, isCheckpointValue } from './custom' -import { splitColumnPath } from '../../experiments/columns/paths' +import { + getFullValuePath, + CustomPlotsOrderValue, + isCheckpointValue +} from './custom' +import { + FILE_SEPARATOR, + splitColumnPath +} from '../../experiments/columns/paths' import { pickFromColumnLikes } from '../../experiments/columns/quickPick' import { Column, ColumnType } from '../../experiments/webview/contract' import { definedAndNonEmpty } from '../../util/array' @@ -14,11 +21,18 @@ import { Toast } from '../../vscode/toast' import { CustomPlotType } from '../webview/contract' const getMetricVsParamPlotItem = (metric: string, param: string) => { - const splitMetric = splitColumnPath(metric) - const splitParam = splitColumnPath(param) + const fullMetric = getFullValuePath( + ColumnType.METRICS, + metric, + FILE_SEPARATOR + ) + const fullParam = getFullValuePath(ColumnType.PARAMS, param, FILE_SEPARATOR) + const splitMetric = splitColumnPath(fullMetric) + const splitParam = splitColumnPath(fullParam) + return { description: 'Metric Vs Param Plot', - detail: `${metric} vs ${param}`, + detail: `${fullMetric} vs ${fullParam}`, label: `${splitMetric[splitMetric.length - 1]} vs ${ splitParam[splitParam.length - 1] }`, @@ -27,10 +41,15 @@ const getMetricVsParamPlotItem = (metric: string, param: string) => { } const getCheckpointPlotItem = (metric: string) => { - const splitMetric = splitColumnPath(metric) + const fullMetric = getFullValuePath( + ColumnType.METRICS, + metric, + FILE_SEPARATOR + ) + const splitMetric = splitColumnPath(fullMetric) return { description: 'Checkpoint Trend Plot', - detail: metric, + detail: fullMetric, label: splitMetric[splitMetric.length - 1], value: getCustomPlotId(metric) } From a804a0c23798fe1d4595240c134110887f23f4cc Mon Sep 17 00:00:00 2001 From: julieg18 Date: Tue, 14 Mar 2023 09:41:05 -0500 Subject: [PATCH 32/40] Fix readme typo --- extension/resources/walkthrough/live-plots.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extension/resources/walkthrough/live-plots.md b/extension/resources/walkthrough/live-plots.md index e945400abd..02bbb88dc1 100644 --- a/extension/resources/walkthrough/live-plots.md +++ b/extension/resources/walkthrough/live-plots.md @@ -33,6 +33,6 @@ for epoch in range(NUM_EPOCHS): `DVCLive` is _optional_, and you can just append or modify plot files using any language and any tool. -💡 Plots created in the `Custom` section of the plots dashboard is being updated -automatically based on the data in the table. You don't even have to manage or -write any special plot files. +💡 Plots created in the `Custom` section of the plots dashboard are being +updated automatically based on the data in the table. You don't even have to +manage or write any special plot files. From 3f309594b1ee666b18984bb5e769a4e77573732b Mon Sep 17 00:00:00 2001 From: julieg18 Date: Tue, 14 Mar 2023 10:58:48 -0500 Subject: [PATCH 33/40] Fix checkpoint spec --- webview/src/plots/components/customPlots/CustomPlot.tsx | 2 +- webview/src/plots/components/customPlots/util.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/webview/src/plots/components/customPlots/CustomPlot.tsx b/webview/src/plots/components/customPlots/CustomPlot.tsx index bd3deda0c1..b2fb2d147e 100644 --- a/webview/src/plots/components/customPlots/CustomPlot.tsx +++ b/webview/src/plots/components/customPlots/CustomPlot.tsx @@ -27,7 +27,7 @@ const createCustomPlotSpec = ( } if (isCheckpointPlot(plot)) { - return createCheckpointSpec(plot.yTitle, plot.param, colors) + return createCheckpointSpec(plot.yTitle, plot.metric, plot.param, colors) } return createMetricVsParamSpec(plot.yTitle, plot.param) } diff --git a/webview/src/plots/components/customPlots/util.ts b/webview/src/plots/components/customPlots/util.ts index f81d3af57b..3e6a3336a4 100644 --- a/webview/src/plots/components/customPlots/util.ts +++ b/webview/src/plots/components/customPlots/util.ts @@ -3,6 +3,7 @@ import { ColorScale } from 'dvc/src/plots/webview/contract' export const createCheckpointSpec = ( title: string, + fullTitle: string, param: string, scale?: ColorScale ): VisualizationSpec => @@ -52,7 +53,7 @@ export const createCheckpointSpec = ( { field: 'group', title: 'name' }, { field: 'y', - title: title.slice(Math.max(0, title.indexOf(':') + 1)), + title: fullTitle.slice(Math.max(0, fullTitle.indexOf(':') + 1)), type: 'quantitative' } ] From 7ebebdf20ba1dce70cc0f039cf679b278328b632 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Fri, 17 Mar 2023 16:31:17 -0500 Subject: [PATCH 34/40] Replace checkpoint with custom in App.test.tsx --- webview/src/plots/components/App.test.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/webview/src/plots/components/App.test.tsx b/webview/src/plots/components/App.test.tsx index bf6c75980f..261a98bc9a 100644 --- a/webview/src/plots/components/App.test.tsx +++ b/webview/src/plots/components/App.test.tsx @@ -684,7 +684,7 @@ describe('App', () => { }) }) - it('should display the checkpoint plots in the order stored', () => { + it('should display the custom plots in the order stored', () => { renderAppWithOptionalData({ comparison: comparisonTableFixture, custom: customPlotsFixture @@ -814,12 +814,12 @@ describe('App', () => { template: templatePlotsFixture }) - const checkpointPlots = screen.getAllByTestId(/summary\.json/) + const customPlots = screen.getAllByTestId(/summary\.json/) const templatePlots = screen.getAllByTestId(/^plot_/) - dragAndDrop(templatePlots[0], checkpointPlots[2]) + dragAndDrop(templatePlots[0], customPlots[2]) - expect(checkpointPlots.map(plot => plot.id)).toStrictEqual([ + expect(customPlots.map(plot => plot.id)).toStrictEqual([ 'custom-summary.json:loss-params.yaml:dropout', 'custom-summary.json:accuracy-params.yaml:epochs', 'custom-summary.json:loss-epoch', @@ -1291,7 +1291,7 @@ describe('App', () => { }) }) - it('should open a modal with the plot zoomed in when clicking a checkpoint plot', () => { + it('should open a modal with the plot zoomed in when clicking a custom plot', () => { renderAppWithOptionalData({ comparison: comparisonTableFixture, custom: customPlotsFixture From 40d0010fc289cc7d1e96f941ca104d24f810d63f Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 20 Mar 2023 09:01:56 -0500 Subject: [PATCH 35/40] Fix "plot already exists" logic --- extension/src/plots/model/custom.ts | 2 +- extension/src/plots/model/quickPick.test.ts | 22 ++++++++++---------- extension/src/plots/model/quickPick.ts | 23 ++++++++++++++++++--- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts index 3f6fcb0a6e..af9747b79b 100644 --- a/extension/src/plots/model/custom.ts +++ b/extension/src/plots/model/custom.ts @@ -25,7 +25,7 @@ export const doesCustomPlotAlreadyExist = ( return value.param === param && value.metric === metric }) -const removeColumnTypeFromPath = ( +export const removeColumnTypeFromPath = ( columnPath: string, type: string, fileSep: string diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index 20f9fbf5a9..2635b5aecf 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -153,7 +153,7 @@ describe('pickMetricAndParam', () => { { hasChildren: false, label: 'accuracy', - path: 'summary.json:accuracy', + path: 'metrics:summary.json:accuracy', type: ColumnType.METRICS } ]) @@ -169,7 +169,7 @@ describe('pickMetricAndParam', () => { { hasChildren: false, label: 'accuracy', - path: 'summary.json:accuracy', + path: 'metrics:summary.json:accuracy', type: ColumnType.METRICS } ]) @@ -183,7 +183,7 @@ describe('pickMetricAndParam', () => { } const expectedParam = { label: 'epochs', - path: 'summary.json:loss-params.yaml:epochs' + path: 'params:params.yaml:epochs' } mockedQuickPickValue .mockResolvedValueOnce(expectedMetric) @@ -200,13 +200,13 @@ describe('pickMetricAndParam', () => { { hasChildren: false, label: 'accuracy', - path: 'summary.json:accuracy', + path: 'metrics:summary.json:accuracy', type: ColumnType.METRICS } ]) expect(metricAndParam).toStrictEqual({ - metric: expectedMetric.path, - param: expectedParam.path + metric: 'summary.json:loss', + param: 'params.yaml:epochs' }) }) }) @@ -232,7 +232,7 @@ describe('pickMetric', () => { { hasChildren: false, label: 'accuracy', - path: 'summary.json:accuracy', + path: 'metrics:summary.json:accuracy', type: ColumnType.METRICS } ]) @@ -250,12 +250,12 @@ describe('pickMetric', () => { { hasChildren: false, label: 'accuracy', - path: 'summary.json:accuracy', + path: 'metrics:summary.json:accuracy', type: ColumnType.METRICS } ]) - expect(metric).toStrictEqual(expectedMetric.path) + expect(metric).toStrictEqual('summary.json:loss') expect(mockedQuickPickValue).toHaveBeenCalledTimes(1) expect(mockedQuickPickValue).toHaveBeenCalledWith( [ @@ -265,9 +265,9 @@ describe('pickMetric', () => { value: { label: 'loss', path: 'metrics:summary.json:loss' } }, { - description: 'summary.json:accuracy', + description: 'metrics:summary.json:accuracy', label: 'accuracy', - value: { label: 'accuracy', path: 'summary.json:accuracy' } + value: { label: 'accuracy', path: 'metrics:summary.json:accuracy' } } ], { title: Title.SELECT_METRIC_CUSTOM_PLOT } diff --git a/extension/src/plots/model/quickPick.ts b/extension/src/plots/model/quickPick.ts index fe491679ed..e9cacd4754 100644 --- a/extension/src/plots/model/quickPick.ts +++ b/extension/src/plots/model/quickPick.ts @@ -2,7 +2,8 @@ import { getCustomPlotId } from './collect' import { getFullValuePath, CustomPlotsOrderValue, - isCheckpointValue + isCheckpointValue, + removeColumnTypeFromPath } from './custom' import { FILE_SEPARATOR, @@ -126,7 +127,19 @@ export const pickMetricAndParam = async (columns: Column[]) => { if (!param) { return } - return { metric: metric.path, param: param.path } + + return { + metric: removeColumnTypeFromPath( + metric.path, + ColumnType.METRICS, + FILE_SEPARATOR + ), + param: removeColumnTypeFromPath( + param.path, + ColumnType.PARAMS, + FILE_SEPARATOR + ) + } } export const pickMetric = async (columns: Column[]) => { @@ -144,5 +157,9 @@ export const pickMetric = async (columns: Column[]) => { return } - return metric.path + return removeColumnTypeFromPath( + metric.path, + ColumnType.METRICS, + FILE_SEPARATOR + ) } From 1ab409c1a7a57cce3a630d2abcd2037031b946a4 Mon Sep 17 00:00:00 2001 From: Julie G <43496356+julieg18@users.noreply.github.com> Date: Mon, 20 Mar 2023 09:47:50 -0500 Subject: [PATCH 36/40] Consolidate `collectCustomPlots` (#3466) --- extension/src/experiments/index.ts | 4 + extension/src/experiments/model/index.ts | 4 + extension/src/plots/index.ts | 9 +- extension/src/plots/model/collect.test.ts | 207 +++------- extension/src/plots/model/collect.ts | 372 ++++++------------ extension/src/plots/model/index.ts | 95 ++--- extension/src/plots/webview/contract.ts | 14 +- extension/src/plots/webview/messages.ts | 14 +- .../test/fixtures/expShow/base/customPlots.ts | 268 ++++++++----- 9 files changed, 393 insertions(+), 594 deletions(-) diff --git a/extension/src/experiments/index.ts b/extension/src/experiments/index.ts index 360af40e31..67f421641f 100644 --- a/extension/src/experiments/index.ts +++ b/extension/src/experiments/index.ts @@ -333,6 +333,10 @@ export class Experiments extends BaseRepository { return this.experiments.getExperimentCount() } + public getExperimentsWithCheckpoints() { + return this.experiments.getExperimentsWithCheckpoints() + } + public async selectExperiments() { const experiments = this.experiments.getExperimentsWithCheckpoints() diff --git a/extension/src/experiments/model/index.ts b/extension/src/experiments/model/index.ts index 0a29502689..1a7a386086 100644 --- a/extension/src/experiments/model/index.ts +++ b/extension/src/experiments/model/index.ts @@ -60,6 +60,10 @@ export type ExperimentWithCheckpoints = Experiment & { checkpoints?: Experiment[] } +export type ExperimentWithDefinedCheckpoints = Experiment & { + checkpoints: Experiment[] +} + export enum ExperimentType { WORKSPACE = 'workspace', COMMIT = 'commit', diff --git a/extension/src/plots/index.ts b/extension/src/plots/index.ts index e612fd456a..d61e04875e 100644 --- a/extension/src/plots/index.ts +++ b/extension/src/plots/index.ts @@ -13,7 +13,6 @@ import { Experiments } from '../experiments' import { Resource } from '../resourceLocator' import { InternalCommands } from '../commands/internal' import { definedAndNonEmpty } from '../util/array' -import { ExperimentsOutput } from '../cli/dvc/contract' import { TEMP_PLOTS_DIR } from '../cli/dvc/constants' import { removeDir } from '../fileSystem' import { Toast } from '../vscode/toast' @@ -173,7 +172,7 @@ export class Plots extends BaseRepository { waitForInitialExpData.dispose() this.data.setMetricFiles(data) this.setupExperimentsListener(experiments) - void this.initializeData(data) + void this.initializeData() } }) ) @@ -184,7 +183,7 @@ export class Plots extends BaseRepository { experiments.onDidChangeExperiments(async data => { if (data) { await Promise.all([ - this.plots.transformAndSetExperiments(data), + this.plots.transformAndSetExperiments(), this.data.setMetricFiles(data) ]) } @@ -200,8 +199,8 @@ export class Plots extends BaseRepository { ) } - private async initializeData(data: ExperimentsOutput) { - await this.plots.transformAndSetExperiments(data) + private async initializeData() { + await this.plots.transformAndSetExperiments() void this.data.managedUpdate() await Promise.all([ this.data.isReady(), diff --git a/extension/src/plots/model/collect.test.ts b/extension/src/plots/model/collect.test.ts index b5175eae4c..295beb3e65 100644 --- a/extension/src/plots/model/collect.test.ts +++ b/extension/src/plots/model/collect.test.ts @@ -1,18 +1,16 @@ import { join } from 'path' -import omit from 'lodash.omit' import isEmpty from 'lodash.isempty' import { collectData, collectTemplates, collectOverrideRevisionDetails, - collectCustomPlots, - collectCustomCheckpointPlots, - collectCustomPlotData + collectCustomPlots } from './collect' +import { isCheckpointPlot } from './custom' import plotsDiffFixture from '../../test/fixtures/plotsDiff/output' import customPlotsFixture, { customPlotsOrderFixture, - checkpointPlotsFixture + experimentsWithCheckpoints } from '../../test/fixtures/expShow/base/customPlots' import { ExperimentStatus, @@ -20,14 +18,13 @@ import { } from '../../cli/dvc/contract' import { sameContents } from '../../util/array' import { - CheckpointPlot, - CustomPlot, CustomPlotData, + CustomPlotType, + DEFAULT_NB_ITEMS_PER_ROW, + DEFAULT_PLOT_HEIGHT, TemplatePlot } from '../webview/contract' import { getCLICommitId } from '../../test/fixtures/plotsDiff/util' -import expShowFixture from '../../test/fixtures/expShow/base/output' -import modifiedFixture from '../../test/fixtures/expShow/modified/output' import { SelectedExperimentWithColor } from '../../experiments/model' import { Experiment } from '../../experiments/webview/contract' @@ -35,120 +32,62 @@ const logsLossPath = join('logs', 'loss.tsv') const logsLossPlot = (plotsDiffFixture[logsLossPath][0] || {}) as TemplatePlot -const getCustomPlotFromCustomPlotData = ({ - id, - metric, - param, - type, - values -}: CustomPlotData) => - ({ - id, - metric, - param, - type, - values - } as CustomPlot) - describe('collectCustomPlots', () => { + const defaultFuncArgs = { + experiments: experimentsWithCheckpoints, + hasCheckpoints: true, + height: DEFAULT_PLOT_HEIGHT, + nbItemsPerRow: DEFAULT_NB_ITEMS_PER_ROW, + plotsOrderValues: customPlotsOrderFixture, + selectedRevisions: customPlotsFixture.colors?.domain + } + it('should return the expected data from the test fixture', () => { - const expectedOutput: CustomPlot[] = customPlotsFixture.plots.map( - getCustomPlotFromCustomPlotData - ) - const data = collectCustomPlots( - customPlotsOrderFixture, - checkpointPlotsFixture, - [ - { - id: '12345', - label: '123', - metrics: { - 'summary.json': { - accuracy: 0.3724166750907898, - loss: 2.0205044746398926 - } - }, - name: 'exp-e7a67', - params: { 'params.yaml': { dropout: 0.15, epochs: 2 } } - }, - { - id: '12345', - label: '123', - metrics: { - 'summary.json': { - accuracy: 0.4668000042438507, - loss: 1.9293040037155151 - } - }, - name: 'test-branch', - params: { 'params.yaml': { dropout: 0.122, epochs: 2 } } - }, - { - id: '12345', - label: '123', - metrics: { - 'summary.json': { - accuracy: 0.5926499962806702, - loss: 1.775016188621521 - } - }, - name: 'exp-83425', - params: { 'params.yaml': { dropout: 0.124, epochs: 5 } } - } - ] + const expectedOutput: CustomPlotData[] = customPlotsFixture.plots + const data = collectCustomPlots(defaultFuncArgs) + expect(data).toStrictEqual(expectedOutput) + }) + + it('should return only custom plots if there no selected revisions', () => { + const expectedOutput: CustomPlotData[] = customPlotsFixture.plots.filter( + plot => plot.type !== CustomPlotType.CHECKPOINT ) + const data = collectCustomPlots({ + ...defaultFuncArgs, + selectedRevisions: undefined + }) expect(data).toStrictEqual(expectedOutput) }) -}) -describe('collectCustomPlotData', () => { - it('should return the expected data from test fixture', () => { - const expectedMetricVsParamPlotData = customPlotsFixture.plots[0] - const expectedCheckpointsPlotData = customPlotsFixture.plots[2] - const metricVsParamPlot = getCustomPlotFromCustomPlotData( - expectedMetricVsParamPlotData - ) - const checkpointsPlot = getCustomPlotFromCustomPlotData( - expectedCheckpointsPlotData + it('should return only custom plots if checkpoints are not enabled', () => { + const expectedOutput: CustomPlotData[] = customPlotsFixture.plots.filter( + plot => plot.type !== CustomPlotType.CHECKPOINT ) + const data = collectCustomPlots({ + ...defaultFuncArgs, + hasCheckpoints: false + }) - const metricVsParamData = collectCustomPlotData( - metricVsParamPlot, - customPlotsFixture.colors, - customPlotsFixture.nbItemsPerRow, - customPlotsFixture.height - ) + expect(data).toStrictEqual(expectedOutput) + }) - const checkpointsData = collectCustomPlotData( - { - ...checkpointsPlot, - values: [ - ...checkpointsPlot.values, - { - group: 'exp-123', - iteration: 1, - y: 1.4534177053451538 - }, - { - group: 'exp-123', - iteration: 2, - y: 1.757687 - }, - { - group: 'exp-123', - iteration: 3, - y: 1.989894 - } - ] - } as CheckpointPlot, - customPlotsFixture.colors, - customPlotsFixture.nbItemsPerRow, - customPlotsFixture.height - ) + it('should return checkpoint plots with values only containing selected experiments data', () => { + const domain = customPlotsFixture.colors?.domain.slice(1) as string[] + + const expectedOutput = customPlotsFixture.plots.map(plot => ({ + ...plot, + values: isCheckpointPlot(plot) + ? plot.values.filter(value => domain.includes(value.group)) + : plot.values + })) - expect(metricVsParamData).toStrictEqual(expectedMetricVsParamPlotData) - expect(checkpointsData).toStrictEqual(expectedCheckpointsPlotData) + const data = collectCustomPlots({ + ...defaultFuncArgs, + selectedRevisions: domain + }) + + expect(data).toStrictEqual(expectedOutput) }) }) @@ -215,50 +154,6 @@ describe('collectData', () => { }) }) -describe('collectCustomCheckpointPlotsData', () => { - it('should return the expected data from the test fixture', () => { - const data = collectCustomCheckpointPlots(expShowFixture) - - expect(data).toStrictEqual(checkpointPlotsFixture) - }) - - it('should provide a continuous series for a modified experiment', () => { - const data = collectCustomCheckpointPlots(modifiedFixture) - - for (const { values } of Object.values(data)) { - const initialExperiment = values.filter( - point => point.group === 'exp-908bd' - ) - const modifiedExperiment = values.find( - point => point.group === 'exp-01b3a' - ) - - const lastIterationInitial = initialExperiment?.slice(-1)[0] - const firstIterationModified = modifiedExperiment - - expect(lastIterationInitial).not.toStrictEqual(firstIterationModified) - expect(omit(lastIterationInitial, 'group')).toStrictEqual( - omit(firstIterationModified, 'group') - ) - - const baseExperiment = values.filter(point => point.group === 'exp-920fc') - const restartedExperiment = values.find( - point => point.group === 'exp-9bc1b' - ) - - const iterationRestartedFrom = baseExperiment?.slice(5)[0] - const firstIterationAfterRestart = restartedExperiment - - expect(iterationRestartedFrom).not.toStrictEqual( - firstIterationAfterRestart - ) - expect(omit(iterationRestartedFrom, 'group')).toStrictEqual( - omit(firstIterationAfterRestart, 'group') - ) - } - }) -}) - describe('collectTemplates', () => { it('should return the expected output from the test fixture', () => { const { content } = logsLossPlot diff --git a/extension/src/plots/model/collect.ts b/extension/src/plots/model/collect.ts index 64b7beccbd..7612e164ae 100644 --- a/extension/src/plots/model/collect.ts +++ b/extension/src/plots/model/collect.ts @@ -1,13 +1,10 @@ -import omit from 'lodash.omit' import get from 'lodash.get' import { TopLevelSpec } from 'vega-lite' import { VisualizationSpec } from 'react-vega' -import { CustomCheckpointPlots } from '.' import { getFullValuePath, CHECKPOINTS_PARAM, CustomPlotsOrderValue, - isCheckpointPlot, isCheckpointValue } from './custom' import { getRevisionFirstThreeColumns } from './util' @@ -22,35 +19,19 @@ import { TemplatePlotSection, PlotsType, Revision, - CustomPlotType, - CustomPlot, - MetricVsParamPlot, - CustomPlotData + CustomPlotData, + MetricVsParamPlotValues } from '../webview/contract' +import { EXPERIMENT_WORKSPACE_ID, PlotsOutput } from '../../cli/dvc/contract' import { - EXPERIMENT_WORKSPACE_ID, - ExperimentFieldsOrError, - ExperimentsOutput, - ExperimentStatus, - isValueTree, - PlotsOutput, - Value, - ValueTree -} from '../../cli/dvc/contract' -import { extractColumns } from '../../experiments/columns/extract' -import { - decodeColumn, - appendColumnToPath, splitColumnPath, FILE_SEPARATOR } from '../../experiments/columns/paths' import { ColumnType, Experiment, - isRunning, - MetricOrParamColumns + isRunning } from '../../experiments/webview/contract' -import { addToMapArray } from '../../util/map' import { TemplateOrder } from '../paths/collect' import { extendVegaSpec, @@ -67,231 +48,72 @@ import { unmergeConcatenatedFields } from '../multiSource/collect' import { StrokeDashEncoding } from '../multiSource/constants' -import { SelectedExperimentWithColor } from '../../experiments/model' +import { + ExperimentWithCheckpoints, + ExperimentWithDefinedCheckpoints, + SelectedExperimentWithColor +} from '../../experiments/model' import { Color } from '../../experiments/model/status/colors' -import { typedValueTreeEntries } from '../../experiments/columns/collect/metricsAndParams' - -type CheckpointPlotAccumulator = { - iterations: Record - plots: Map -} - -const collectFromMetricsFile = ( - acc: CheckpointPlotAccumulator, - name: string, - iteration: number, - key: string | undefined, - value: Value | ValueTree, - ancestors: string[] = [] -) => { - const pathArray = [...ancestors, key].filter(Boolean) as string[] - - if (isValueTree(value)) { - for (const [childKey, childValue] of typedValueTreeEntries(value)) { - collectFromMetricsFile( - acc, - name, - iteration, - childKey, - childValue, - pathArray - ) - } - return - } - - const path = appendColumnToPath(...pathArray) - - addToMapArray(acc.plots, path, { group: name, iteration, y: value }) -} - -type MetricsAndDetailsOrUndefined = - | { - checkpoint_parent: string | undefined - checkpoint_tip: string | undefined - metrics: MetricOrParamColumns | undefined - status: ExperimentStatus | undefined - } - | undefined - -const transformExperimentData = ( - experimentFieldsOrError: ExperimentFieldsOrError -): MetricsAndDetailsOrUndefined => { - const experimentFields = experimentFieldsOrError.data - if (!experimentFields) { - return - } - - const { checkpoint_tip, checkpoint_parent, status } = experimentFields - const { metrics } = extractColumns(experimentFields) - - return { checkpoint_parent, checkpoint_tip, metrics, status } -} - -type ValidData = { - checkpoint_parent: string - checkpoint_tip: string - metrics: MetricOrParamColumns - status: ExperimentStatus -} -const isValid = (data: MetricsAndDetailsOrUndefined): data is ValidData => - !!(data?.checkpoint_tip && data?.checkpoint_parent && data?.metrics) +export const getCustomPlotId = (metric: string, param = CHECKPOINTS_PARAM) => + `custom-${metric}-${param}` -const collectFromMetrics = ( - acc: CheckpointPlotAccumulator, - experimentName: string, - iteration: number, - metrics: MetricOrParamColumns -) => { - for (const file of Object.keys(metrics)) { - collectFromMetricsFile( - acc, - experimentName, - iteration, - undefined, - metrics[file], - [file] - ) - } -} +const getValueFromColumn = ( + path: string, + experiment: ExperimentWithCheckpoints +) => get(experiment, splitColumnPath(path)) as number | undefined -const getLastIteration = ( - acc: CheckpointPlotAccumulator, - checkpointParent: string -): number => acc.iterations[checkpointParent] || 0 - -const collectIteration = ( - acc: CheckpointPlotAccumulator, - sha: string, - checkpointParent: string -): number => { - const iteration = getLastIteration(acc, checkpointParent) + 1 - acc.iterations[sha] = iteration - return iteration -} +const isExperimentWithDefinedCheckpoints = ( + experiment: ExperimentWithCheckpoints +): experiment is ExperimentWithDefinedCheckpoints => !!experiment.checkpoints -const linkModified = ( - acc: CheckpointPlotAccumulator, - experimentName: string, - checkpointTip: string, - checkpointParent: string, - parent: ExperimentFieldsOrError | undefined +const collectCheckpointValuesFromExperiment = ( + values: CheckpointPlotValues, + exp: ExperimentWithDefinedCheckpoints, + metricPath: string ) => { - if (!parent) { - return - } + const group = exp.name || exp.label + const maxEpoch = exp.checkpoints.length + 1 - const parentData = transformExperimentData(parent) - if (!isValid(parentData) || parentData.checkpoint_tip === checkpointTip) { - return + const metricValue = getValueFromColumn(metricPath, exp) + if (metricValue !== undefined) { + values.push({ group, iteration: maxEpoch, y: metricValue }) } - const lastIteration = getLastIteration(acc, checkpointParent) - collectFromMetrics(acc, experimentName, lastIteration, parentData.metrics) -} - -const collectFromExperimentsObject = ( - acc: CheckpointPlotAccumulator, - experimentsObject: { [sha: string]: ExperimentFieldsOrError } -) => { - for (const [sha, experimentData] of Object.entries( - experimentsObject - ).reverse()) { - const data = transformExperimentData(experimentData) - - if (!isValid(data)) { - continue + for (const [ind, checkpoint] of exp.checkpoints.entries()) { + const metricValue = getValueFromColumn(metricPath, checkpoint) + if (metricValue !== undefined) { + values.push({ group, iteration: maxEpoch - ind - 1, y: metricValue }) } - const { - checkpoint_tip: checkpointTip, - checkpoint_parent: checkpointParent, - metrics - } = data - - const experimentName = experimentsObject[checkpointTip].data?.name - if (!experimentName) { - continue - } - - linkModified( - acc, - experimentName, - checkpointTip, - checkpointParent, - experimentsObject[checkpointParent] - ) - - const iteration = collectIteration(acc, sha, checkpointParent) - collectFromMetrics(acc, experimentName, iteration, metrics) } } -export const getCustomPlotId = (metric: string, param = CHECKPOINTS_PARAM) => - `custom-${metric}-${param}` - -export const collectCustomCheckpointPlots = ( - data: ExperimentsOutput -): CustomCheckpointPlots => { - const acc = { - iterations: {}, - plots: new Map() - } - - for (const { baseline, ...experimentsObject } of Object.values( - omit(data, EXPERIMENT_WORKSPACE_ID) - )) { - const commit = transformExperimentData(baseline) - - if (commit) { - collectFromExperimentsObject(acc, experimentsObject) - } - } - - const plotsData: CustomCheckpointPlots = {} - if (acc.plots.size === 0) { - return plotsData - } - - for (const [key, value] of acc.plots.entries()) { - const decodedMetric = decodeColumn(key) - plotsData[decodedMetric] = { - id: getCustomPlotId(decodedMetric), - metric: decodedMetric, - param: CHECKPOINTS_PARAM, - type: CustomPlotType.CHECKPOINT, - values: value +const getCheckpointValues = ( + experiments: ExperimentWithCheckpoints[], + metricPath: string +): CheckpointPlotValues => { + const values: CheckpointPlotValues = [] + for (const experiment of experiments) { + if (isExperimentWithDefinedCheckpoints(experiment)) { + collectCheckpointValuesFromExperiment(values, experiment, metricPath) } } - - return plotsData + return values } -const collectMetricVsParamPlot = ( - metric: string, - param: string, - experiments: Experiment[] -): MetricVsParamPlot => { - const splitUpMetricPath = splitColumnPath( - getFullValuePath(ColumnType.METRICS, metric, FILE_SEPARATOR) - ) - const splitUpParamPath = splitColumnPath( - getFullValuePath(ColumnType.PARAMS, param, FILE_SEPARATOR) - ) - const plotData: MetricVsParamPlot = { - id: getCustomPlotId(metric, param), - metric, - param, - type: CustomPlotType.METRIC_VS_PARAM, - values: [] - } +const getMetricVsParamValues = ( + experiments: ExperimentWithCheckpoints[], + metricPath: string, + paramPath: string +): MetricVsParamPlotValues => { + const values: MetricVsParamPlotValues = [] for (const experiment of experiments) { - const metricValue = get(experiment, splitUpMetricPath) as number | undefined - const paramValue = get(experiment, splitUpParamPath) as number | undefined + const metricValue = getValueFromColumn(metricPath, experiment) + const paramValue = getValueFromColumn(paramPath, experiment) if (metricValue !== undefined && paramValue !== undefined) { - plotData.values.push({ + values.push({ expName: experiment.name || experiment.label, metric: metricValue, param: paramValue @@ -299,44 +121,78 @@ const collectMetricVsParamPlot = ( } } - return plotData + return values } -export const collectCustomPlots = ( - plotsOrderValues: CustomPlotsOrderValue[], - checkpointPlots: CustomCheckpointPlots, - experiments: Experiment[] -): CustomPlot[] => { - return plotsOrderValues - .map((plotOrderValue): CustomPlot => { - if (isCheckpointValue(plotOrderValue.type)) { - const { metric } = plotOrderValue - return checkpointPlots[metric] - } - const { metric, param } = plotOrderValue - return collectMetricVsParamPlot(metric, param, experiments) - }) - .filter(Boolean) +const getCustomPlotData = ( + orderValue: CustomPlotsOrderValue, + experiments: ExperimentWithCheckpoints[], + selectedRevisions: string[] | undefined = [], + height: number, + nbItemsPerRow: number +): CustomPlotData => { + const { metric, param, type } = orderValue + const metricPath = getFullValuePath( + ColumnType.METRICS, + metric, + FILE_SEPARATOR + ) + + const paramPath = getFullValuePath(ColumnType.PARAMS, param, FILE_SEPARATOR) + + const selectedExperiments = experiments.filter(({ name, label }) => + selectedRevisions.includes(name || label) + ) + + const values = isCheckpointValue(type) + ? getCheckpointValues(selectedExperiments, metricPath) + : getMetricVsParamValues(experiments, metricPath, paramPath) + + return { + id: getCustomPlotId(metric, param), + metric, + param, + type, + values, + yTitle: truncateVerticalTitle(metric, nbItemsPerRow, height) as string + } as CustomPlotData } -export const collectCustomPlotData = ( - plot: CustomPlot, - colors: ColorScale | undefined, - nbItemsPerRow: number, +export const collectCustomPlots = ({ + plotsOrderValues, + experiments, + hasCheckpoints, + selectedRevisions, + height, + nbItemsPerRow +}: { + plotsOrderValues: CustomPlotsOrderValue[] + experiments: ExperimentWithCheckpoints[] + hasCheckpoints: boolean + selectedRevisions: string[] | undefined height: number -): CustomPlotData => { - const selectedExperiments = colors?.domain - const filteredValues = isCheckpointPlot(plot) - ? plot.values.filter(value => - (selectedExperiments as string[]).includes(value.group) + nbItemsPerRow: number +}): CustomPlotData[] => { + const plots = [] + const shouldSkipCheckpointPlots = !hasCheckpoints || !selectedRevisions + + for (const value of plotsOrderValues) { + if (shouldSkipCheckpointPlots && isCheckpointValue(value.type)) { + continue + } + + plots.push( + getCustomPlotData( + value, + experiments, + selectedRevisions, + height, + nbItemsPerRow ) - : plot.values + ) + } - return { - ...plot, - values: filteredValues, - yTitle: truncateVerticalTitle(plot.metric, nbItemsPerRow, height) as string - } as CustomPlotData + return plots } type RevisionPathData = { [path: string]: Record[] } diff --git a/extension/src/plots/model/index.ts b/extension/src/plots/model/index.ts index e8a8a1bd05..118c51de26 100644 --- a/extension/src/plots/model/index.ts +++ b/extension/src/plots/model/index.ts @@ -11,16 +11,10 @@ import { collectCommitRevisionDetails, collectOverrideRevisionDetails, collectCustomPlots, - getCustomPlotId, - collectCustomCheckpointPlots, - collectCustomPlotData + getCustomPlotId } from './collect' import { getRevisionFirstThreeColumns } from './util' -import { - cleanupOldOrderValue, - CustomPlotsOrderValue, - isCheckpointPlot -} from './custom' +import { cleanupOldOrderValue, CustomPlotsOrderValue } from './custom' import { CheckpointPlot, ComparisonPlots, @@ -32,14 +26,11 @@ import { SectionCollapsed, CustomPlotData, CustomPlotsData, - CustomPlot, - ColorScale, DEFAULT_HEIGHT, DEFAULT_NB_ITEMS_PER_ROW, PlotHeight } from '../webview/contract' import { - ExperimentsOutput, EXPERIMENT_WORKSPACE_ID, PlotsOutputOrError } from '../../cli/dvc/contract' @@ -80,9 +71,6 @@ export class PlotsModel extends ModelWithPersistence { private multiSourceVariations: MultiSourceVariations = {} private multiSourceEncoding: MultiSourceEncoding = {} - private customCheckpointPlots?: CustomCheckpointPlots - private customPlots?: CustomPlot[] - constructor( dvcRoot: string, experiments: Experiments, @@ -105,9 +93,7 @@ export class PlotsModel extends ModelWithPersistence { this.customPlotsOrder = this.revive(PersistenceKey.PLOTS_CUSTOM_ORDER, []) } - public transformAndSetExperiments(data: ExperimentsOutput) { - this.recreateCustomPlots(data) - + public transformAndSetExperiments() { return this.removeStaleData() } @@ -124,7 +110,6 @@ export class PlotsModel extends ModelWithPersistence { collectTemplates(data), collectMultiSourceVariations(data, this.multiSourceVariations) ]) - this.recreateCustomPlots() this.comparisonData = { ...this.comparisonData, @@ -153,7 +138,13 @@ export class PlotsModel extends ModelWithPersistence { } public getCustomPlots(): CustomPlotsData | undefined { - if (!this.customPlots) { + const experimentsWithNoCommitData = this.experiments.hasCheckpoints() + ? this.experiments + .getExperimentsWithCheckpoints() + .filter(({ checkpoints }) => !!checkpoints) + : this.experiments.getExperiments() + + if (experimentsWithNoCommitData.length === 0) { return } @@ -162,32 +153,31 @@ export class PlotsModel extends ModelWithPersistence { .getSelectedExperiments() .map(({ displayColor, id: revision }) => ({ displayColor, revision })) ) + const height = this.getHeight(PlotsSection.CUSTOM_PLOTS) + const nbItemsPerRow = this.getNbItemsPerRowOrWidth( + PlotsSection.CUSTOM_PLOTS + ) + const plotsOrderValues = this.getCustomPlotsOrder() + + const plots: CustomPlotData[] = collectCustomPlots({ + experiments: experimentsWithNoCommitData, + hasCheckpoints: this.experiments.hasCheckpoints(), + height, + nbItemsPerRow, + plotsOrderValues, + selectedRevisions: colors?.domain + }) - return { - colors, - height: this.getHeight(PlotsSection.CUSTOM_PLOTS), - nbItemsPerRow: this.getNbItemsPerRowOrWidth(PlotsSection.CUSTOM_PLOTS), - plots: this.getCustomPlotsData(this.customPlots, colors) - } - } - - public recreateCustomPlots(data?: ExperimentsOutput) { - if (data) { - this.customCheckpointPlots = collectCustomCheckpointPlots(data) + if (plots.length === 0 && plotsOrderValues.length > 0) { + return } - const experiments = this.experiments.getExperiments() - - if (experiments.length === 0) { - this.customPlots = undefined - return + return { + colors, + height, + nbItemsPerRow, + plots } - const customPlots: CustomPlot[] = collectCustomPlots( - this.getCustomPlotsOrder(), - this.customCheckpointPlots || {}, - experiments - ) - this.customPlots = customPlots } public getCustomPlotsOrder() { @@ -198,7 +188,6 @@ export class PlotsModel extends ModelWithPersistence { public updateCustomPlotsOrder(plotsOrder: CustomPlotsOrderValue[]) { this.customPlotsOrder = plotsOrder - this.recreateCustomPlots() } public setCustomPlotsOrder(plotsOrder: CustomPlotsOrderValue[]) { @@ -461,28 +450,6 @@ export class PlotsModel extends ModelWithPersistence { return this.commitRevisions[label] || label } - private getCustomPlotsData( - plots: CustomPlot[], - colors: ColorScale | undefined - ): CustomPlotData[] { - const selectedExperimentsExist = !!colors - const filteredPlots: CustomPlotData[] = [] - for (const plot of plots) { - if (!selectedExperimentsExist && isCheckpointPlot(plot)) { - continue - } - filteredPlots.push( - collectCustomPlotData( - plot, - colors, - this.getNbItemsPerRowOrWidth(PlotsSection.CUSTOM_PLOTS), - this.getHeight(PlotsSection.CUSTOM_PLOTS) - ) - ) - } - return filteredPlots - } - private getSelectedComparisonPlots( paths: string[], selectedRevisions: string[] diff --git a/extension/src/plots/webview/contract.ts b/extension/src/plots/webview/contract.ts index 679fa40f3d..6dcadff4d2 100644 --- a/extension/src/plots/webview/contract.ts +++ b/extension/src/plots/webview/contract.ts @@ -91,21 +91,21 @@ export type CheckpointPlotValues = { export type ColorScale = { domain: string[]; range: Color[] } -export type CheckpointPlot = { +type CustomPlotBase = { id: string - values: CheckpointPlotValues metric: string param: string - type: CustomPlotType.CHECKPOINT } +export type CheckpointPlot = { + values: CheckpointPlotValues + type: CustomPlotType.CHECKPOINT +} & CustomPlotBase + export type MetricVsParamPlot = { - id: string values: MetricVsParamPlotValues - metric: string - param: string type: CustomPlotType.METRIC_VS_PARAM -} +} & CustomPlotBase export type CustomPlot = MetricVsParamPlot | CheckpointPlot diff --git a/extension/src/plots/webview/messages.ts b/extension/src/plots/webview/messages.ts index 85f724aca8..4c64602ab4 100644 --- a/extension/src/plots/webview/messages.ts +++ b/extension/src/plots/webview/messages.ts @@ -37,6 +37,7 @@ import { doesCustomPlotAlreadyExist, isCheckpointValue } from '../model/custom' +import { getCustomPlotId } from '../model/collect' export class WebviewMessages { private readonly paths: PathsModel @@ -282,20 +283,23 @@ export class WebviewMessages { } private setCustomPlotsOrder(plotIds: string[]) { - const customPlots = this.plots.getCustomPlots()?.plots - if (!customPlots) { - return - } + const customPlotsOrderWithId = this.plots + .getCustomPlotsOrder() + .map(value => ({ + ...value, + id: getCustomPlotId(value.metric, value.param) + })) const newOrder: CustomPlotsOrderValue[] = reorderObjectList( plotIds, - customPlots, + customPlotsOrderWithId, 'id' ).map(({ metric, param, type }) => ({ metric, param, type })) + this.plots.setCustomPlotsOrder(newOrder) this.sendCustomPlotsAndEvent(EventName.VIEWS_REORDER_PLOTS_CUSTOM) } diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index f13bdaf77c..37536b2a2f 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -1,5 +1,5 @@ +import { ExperimentWithCheckpoints } from '../../../../experiments/model' import { copyOriginalColors } from '../../../../experiments/model/status/colors' -import { CustomCheckpointPlots } from '../../../../plots/model' import { CHECKPOINTS_PARAM, CustomPlotsOrderValue @@ -34,88 +34,158 @@ export const customPlotsOrderFixture: CustomPlotsOrderValue[] = [ } ] -export const checkpointPlotsFixture: CustomCheckpointPlots = { - 'summary.json:loss': { - id: 'custom-summary.json:loss-epoch', - metric: 'summary.json:loss', - param: CHECKPOINTS_PARAM, - type: CustomPlotType.CHECKPOINT, - values: [ - { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, - { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, - { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, - { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, - { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, - { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, - { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, - { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, - { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, - { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, - { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, - { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } - ] - }, - 'summary.json:accuracy': { - id: 'custom-summary.json:accuracy-epoch', - metric: 'summary.json:accuracy', - param: CHECKPOINTS_PARAM, - type: CustomPlotType.CHECKPOINT, - values: [ - { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, - { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, - { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, - { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, - { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, - { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, - { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, - { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, - { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, - { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, - { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, - { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } +export const experimentsWithCheckpoints: ExperimentWithCheckpoints[] = [ + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.3724166750907898, + loss: 2.0205044746398926 + } + }, + name: 'exp-e7a67', + params: { 'params.yaml': { dropout: 0.15, epochs: 2 } }, + checkpoints: [ + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.3724166750907898, + loss: 2.0205044746398926 + } + }, + name: 'exp-e7a67', + params: { 'params.yaml': { dropout: 0.15, epochs: 2 } } + }, + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.3723166584968567, + loss: 2.020392894744873 + } + }, + name: 'exp-e7a67', + params: { 'params.yaml': { dropout: 0.15, epochs: 2 } } + } ] }, - 'summary.json:val_loss': { - id: 'custom-summary.json:val_loss-epoch', - metric: 'summary.json:val_loss', - param: CHECKPOINTS_PARAM, - type: CustomPlotType.CHECKPOINT, - values: [ - { group: 'exp-83425', iteration: 1, y: 1.9391471147537231 }, - { group: 'exp-83425', iteration: 2, y: 1.8825950622558594 }, - { group: 'exp-83425', iteration: 3, y: 1.827923059463501 }, - { group: 'exp-83425', iteration: 4, y: 1.7749212980270386 }, - { group: 'exp-83425', iteration: 5, y: 1.7233840227127075 }, - { group: 'exp-83425', iteration: 6, y: 1.7233840227127075 }, - { group: 'test-branch', iteration: 1, y: 1.9363881349563599 }, - { group: 'test-branch', iteration: 2, y: 1.8770883083343506 }, - { group: 'test-branch', iteration: 3, y: 1.8770883083343506 }, - { group: 'exp-e7a67', iteration: 1, y: 1.9979370832443237 }, - { group: 'exp-e7a67', iteration: 2, y: 1.9979370832443237 }, - { group: 'exp-e7a67', iteration: 3, y: 1.9979370832443237 } + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.4668000042438507, + loss: 1.9293040037155151 + } + }, + name: 'test-branch', + params: { 'params.yaml': { dropout: 0.122, epochs: 2 } }, + checkpoints: [ + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.4668000042438507, + loss: 1.9293040037155151 + } + }, + name: 'test-branch', + params: { 'params.yaml': { dropout: 0.122, epochs: 2 } } + }, + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.4083833396434784, + loss: 1.9882521629333496 + } + }, + name: 'test-branch', + params: { 'params.yaml': { dropout: 0.122, epochs: 2 } } + } ] }, - 'summary.json:val_accuracy': { - id: 'custom-summary.json:val_accuracy-epoch', - metric: 'summary.json:val_accuracy', - param: CHECKPOINTS_PARAM, - type: CustomPlotType.CHECKPOINT, - values: [ - { group: 'exp-83425', iteration: 1, y: 0.49399998784065247 }, - { group: 'exp-83425', iteration: 2, y: 0.5550000071525574 }, - { group: 'exp-83425', iteration: 3, y: 0.6035000085830688 }, - { group: 'exp-83425', iteration: 4, y: 0.6414999961853027 }, - { group: 'exp-83425', iteration: 5, y: 0.6704000234603882 }, - { group: 'exp-83425', iteration: 6, y: 0.6704000234603882 }, - { group: 'test-branch', iteration: 1, y: 0.4970000088214874 }, - { group: 'test-branch', iteration: 2, y: 0.5608000159263611 }, - { group: 'test-branch', iteration: 3, y: 0.5608000159263611 }, - { group: 'exp-e7a67', iteration: 1, y: 0.4277999997138977 }, - { group: 'exp-e7a67', iteration: 2, y: 0.4277999997138977 }, - { group: 'exp-e7a67', iteration: 3, y: 0.4277999997138977 } + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.5926499962806702, + loss: 1.775016188621521 + } + }, + name: 'exp-83425', + params: { 'params.yaml': { dropout: 0.124, epochs: 5 } }, + checkpoints: [ + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.5926499962806702, + loss: 1.775016188621521 + } + }, + name: 'exp-83425', + params: { 'params.yaml': { dropout: 0.124, epochs: 5 } } + }, + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.557449996471405, + loss: 1.8261293172836304 + } + }, + name: 'exp-83425', + params: { 'params.yaml': { dropout: 0.124, epochs: 5 } } + }, + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.5113166570663452, + loss: 1.8798457384109497 + } + }, + name: 'exp-83425', + params: { 'params.yaml': { dropout: 0.124, epochs: 5 } } + }, + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.46094998717308044, + loss: 1.9329891204833984 + } + }, + name: 'exp-83425', + params: { 'params.yaml': { dropout: 0.124, epochs: 5 } } + }, + { + id: '12345', + label: '123', + metrics: { + 'summary.json': { + accuracy: 0.40904998779296875, + loss: 1.9896177053451538 + } + }, + name: 'exp-83425', + params: { 'params.yaml': { dropout: 0.124, epochs: 5 } } + } ] } -} +] const colors = copyOriginalColors() @@ -178,18 +248,18 @@ const data: CustomPlotsData = { metric: 'summary.json:loss', param: CHECKPOINTS_PARAM, values: [ - { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 }, - { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, - { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, - { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, - { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, - { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, - { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, - { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, - { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, - { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, + { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 }, { group: 'exp-e7a67', iteration: 2, y: 2.0205044746398926 }, - { group: 'exp-e7a67', iteration: 3, y: 2.0205044746398926 } + { group: 'exp-e7a67', iteration: 1, y: 2.020392894744873 }, + { group: 'test-branch', iteration: 3, y: 1.9293040037155151 }, + { group: 'test-branch', iteration: 2, y: 1.9293040037155151 }, + { group: 'test-branch', iteration: 1, y: 1.9882521629333496 }, + { group: 'exp-83425', iteration: 6, y: 1.775016188621521 }, + { group: 'exp-83425', iteration: 5, y: 1.775016188621521 }, + { group: 'exp-83425', iteration: 4, y: 1.8261293172836304 }, + { group: 'exp-83425', iteration: 3, y: 1.8798457384109497 }, + { group: 'exp-83425', iteration: 2, y: 1.9329891204833984 }, + { group: 'exp-83425', iteration: 1, y: 1.9896177053451538 } ], type: CustomPlotType.CHECKPOINT, yTitle: 'summary.json:loss' @@ -199,18 +269,18 @@ const data: CustomPlotsData = { metric: 'summary.json:accuracy', param: CHECKPOINTS_PARAM, values: [ - { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 }, - { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, - { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, - { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, - { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, - { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, - { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, - { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, - { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, - { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, + { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 }, { group: 'exp-e7a67', iteration: 2, y: 0.3724166750907898 }, - { group: 'exp-e7a67', iteration: 3, y: 0.3724166750907898 } + { group: 'exp-e7a67', iteration: 1, y: 0.3723166584968567 }, + { group: 'test-branch', iteration: 3, y: 0.4668000042438507 }, + { group: 'test-branch', iteration: 2, y: 0.4668000042438507 }, + { group: 'test-branch', iteration: 1, y: 0.4083833396434784 }, + { group: 'exp-83425', iteration: 6, y: 0.5926499962806702 }, + { group: 'exp-83425', iteration: 5, y: 0.5926499962806702 }, + { group: 'exp-83425', iteration: 4, y: 0.557449996471405 }, + { group: 'exp-83425', iteration: 3, y: 0.5113166570663452 }, + { group: 'exp-83425', iteration: 2, y: 0.46094998717308044 }, + { group: 'exp-83425', iteration: 1, y: 0.40904998779296875 } ], type: CustomPlotType.CHECKPOINT, yTitle: 'summary.json:accuracy' From a2b9c75787e3329356324aa2de6844f7b1fb29c5 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 20 Mar 2023 10:32:33 -0500 Subject: [PATCH 37/40] Add tests for custom.ts --- extension/src/plots/model/custom.test.ts | 86 +++++++++++++++++++++ extension/src/plots/model/custom.ts | 8 +- extension/src/plots/model/quickPick.test.ts | 12 +++ 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 extension/src/plots/model/custom.test.ts diff --git a/extension/src/plots/model/custom.test.ts b/extension/src/plots/model/custom.test.ts new file mode 100644 index 0000000000..e1aab504c7 --- /dev/null +++ b/extension/src/plots/model/custom.test.ts @@ -0,0 +1,86 @@ +import { + CHECKPOINTS_PARAM, + cleanupOldOrderValue, + doesCustomPlotAlreadyExist +} from './custom' +import { CustomPlotType } from '../webview/contract' +import { FILE_SEPARATOR } from '../../experiments/columns/paths' + +describe('doesCustomPlotAlreadyExist', () => { + it('should return true if plot exists', () => { + const output = doesCustomPlotAlreadyExist( + [ + { + metric: 'summary.json:loss', + param: 'params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + }, + { + metric: 'summary.json:accuracy', + param: 'params.yaml:epochs', + type: CustomPlotType.METRIC_VS_PARAM + }, + { + metric: 'summary.json:loss', + param: CHECKPOINTS_PARAM, + type: CustomPlotType.CHECKPOINT + } + ], + 'summary.json:accuracy', + 'params.yaml:epochs' + ) + expect(output).toStrictEqual(true) + }) + + it('should return false if plot does not exists', () => { + const output = doesCustomPlotAlreadyExist( + [ + { + metric: 'summary.json:loss', + param: 'params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + }, + { + metric: 'summary.json:accuracy', + param: 'params.yaml:epochs', + type: CustomPlotType.METRIC_VS_PARAM + }, + { + metric: 'summary.json:loss', + param: CHECKPOINTS_PARAM, + type: CustomPlotType.CHECKPOINT + } + ], + 'summary.json:loss', + 'params.yaml:epochs' + ) + expect(output).toStrictEqual(false) + }) +}) + +describe('cleanupOlderValue', () => { + it('should update value if contents are outdated', () => { + const output = cleanupOldOrderValue( + { + metric: 'metrics:summary.json:loss', + param: 'params:params.yaml:dropout' + }, + FILE_SEPARATOR + ) + expect(output).toStrictEqual({ + metric: 'summary.json:loss', + param: 'params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + }) + }) + + it('should not update value if contents are not outdated', () => { + const value = { + metric: 'summary.json:loss', + param: 'params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + } + const output = cleanupOldOrderValue(value, FILE_SEPARATOR) + expect(output).toStrictEqual(value) + }) +}) diff --git a/extension/src/plots/model/custom.ts b/extension/src/plots/model/custom.ts index af9747b79b..10a0e7e783 100644 --- a/extension/src/plots/model/custom.ts +++ b/extension/src/plots/model/custom.ts @@ -41,12 +41,12 @@ export const getFullValuePath = ( ) => type + fileSep + columnPath export const cleanupOldOrderValue = ( - { param, metric, type }: CustomPlotsOrderValue, + value: { metric: string; param: string } | CustomPlotsOrderValue, fileSep: string ): CustomPlotsOrderValue => ({ // previous column paths have the "TYPE:" prefix - metric: removeColumnTypeFromPath(metric, ColumnType.METRICS, fileSep), - param: removeColumnTypeFromPath(param, ColumnType.PARAMS, fileSep), + metric: removeColumnTypeFromPath(value.metric, ColumnType.METRICS, fileSep), + param: removeColumnTypeFromPath(value.param, ColumnType.PARAMS, fileSep), // previous values didn't have a type - type: type || CustomPlotType.METRIC_VS_PARAM + type: (value as CustomPlotsOrderValue).type || CustomPlotType.METRIC_VS_PARAM }) diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index 2635b5aecf..b2d2b4c438 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -229,6 +229,18 @@ describe('pickMetric', () => { path: 'params:params.yaml:dropout', type: ColumnType.PARAMS }, + { + hasChildren: false, + label: 'dropout', + path: 'params:params.yaml:epochs', + type: ColumnType.PARAMS + }, + { + hasChildren: false, + label: 'accuracy', + path: 'metrics:summary.json:loss', + type: ColumnType.METRICS + }, { hasChildren: false, label: 'accuracy', From 418ee5316c393b2e776d26471d8c51f6869e64db Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 20 Mar 2023 10:59:00 -0500 Subject: [PATCH 38/40] Fix typo in tests --- extension/src/plots/model/quickPick.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/src/plots/model/quickPick.test.ts b/extension/src/plots/model/quickPick.test.ts index b2d2b4c438..c73dca9695 100644 --- a/extension/src/plots/model/quickPick.test.ts +++ b/extension/src/plots/model/quickPick.test.ts @@ -222,7 +222,7 @@ describe('pickMetric', () => { it('should end early if user does not select a metric', async () => { mockedQuickPickValue.mockResolvedValue(undefined) - const noMetricSelected = await pickMetricAndParam([ + const noMetricSelected = await pickMetric([ { hasChildren: false, label: 'dropout', From bd305a1f1fe0153e185d1db70293acd62fd23ab8 Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 20 Mar 2023 12:07:47 -0500 Subject: [PATCH 39/40] Add tests for user ending early on +/- custom plots --- extension/src/test/suite/plots/index.test.ts | 130 +++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/extension/src/test/suite/plots/index.test.ts b/extension/src/test/suite/plots/index.test.ts index 2e664a6f29..8ba89d0b63 100644 --- a/extension/src/test/suite/plots/index.test.ts +++ b/extension/src/test/suite/plots/index.test.ts @@ -928,6 +928,92 @@ suite('Plots Test Suite', () => { ) }) + it('should handle a add custom plot message when user ends early', async () => { + const { plots, plotsModel } = await buildPlots( + disposable, + plotsDiffFixture + ) + + const webview = await plots.showWebview() + + const mockPickCustomPlotType = stub( + customPlotQuickPickUtil, + 'pickCustomPlotType' + ) + + const mockGetMetricAndParam = stub( + customPlotQuickPickUtil, + 'pickMetricAndParam' + ) + const mockGetMetric = stub(customPlotQuickPickUtil, 'pickMetric') + + const pickUndefinedType = new Promise(resolve => + mockPickCustomPlotType.onFirstCall().callsFake(() => { + resolve(undefined) + + return Promise.resolve(undefined) + }) + ) + + const mockSetCustomPlotsOrder = stub(plotsModel, 'setCustomPlotsOrder') + mockSetCustomPlotsOrder.returns(undefined) + + const mockSendTelemetryEvent = stub(Telemetry, 'sendTelemetryEvent') + const mockMessageReceived = getMessageReceivedEmitter(webview) + + mockMessageReceived.fire({ type: MessageFromWebviewType.ADD_CUSTOM_PLOT }) + + await pickUndefinedType + + expect(mockSetCustomPlotsOrder).to.not.be.called + expect(mockSendTelemetryEvent).to.not.be.called + + const pickMetricVsParamType = new Promise(resolve => + mockPickCustomPlotType.onSecondCall().callsFake(() => { + resolve(undefined) + + return Promise.resolve(CustomPlotType.METRIC_VS_PARAM) + }) + ) + + const pickMetricVsParamUndefOptions = new Promise(resolve => + mockGetMetricAndParam.onFirstCall().callsFake(() => { + resolve(undefined) + return Promise.resolve(undefined) + }) + ) + + mockMessageReceived.fire({ type: MessageFromWebviewType.ADD_CUSTOM_PLOT }) + + await pickMetricVsParamType + await pickMetricVsParamUndefOptions + + expect(mockSetCustomPlotsOrder).to.not.be.called + expect(mockSendTelemetryEvent).to.not.be.called + + const pickCheckpointType = new Promise(resolve => + mockPickCustomPlotType.onThirdCall().callsFake(() => { + resolve(undefined) + + return Promise.resolve(CustomPlotType.CHECKPOINT) + }) + ) + const pickCheckpointUndefOptions = new Promise(resolve => + mockGetMetric.onFirstCall().callsFake(() => { + resolve(undefined) + return Promise.resolve(undefined) + }) + ) + + mockMessageReceived.fire({ type: MessageFromWebviewType.ADD_CUSTOM_PLOT }) + + await pickCheckpointType + await pickCheckpointUndefOptions + + expect(mockSetCustomPlotsOrder).to.not.be.called + expect(mockSendTelemetryEvent).to.not.be.called + }) + it('should handle a remove custom plot message from the webview', async () => { const { plots, plotsModel } = await buildPlots( disposable, @@ -976,5 +1062,49 @@ suite('Plots Test Suite', () => { undefined ) }) + + it('should handle a remove custom plot message from the webview when user ends early', async () => { + const { plots, plotsModel } = await buildPlots( + disposable, + plotsDiffFixture + ) + + const webview = await plots.showWebview() + + const mockSelectCustomPlots = stub( + customPlotQuickPickUtil, + 'pickCustomPlots' + ) + + const quickPickEvent = new Promise(resolve => + mockSelectCustomPlots.callsFake(() => { + resolve(undefined) + return Promise.resolve(undefined) + }) + ) + + stub(plotsModel, 'getCustomPlotsOrder').returns([ + { + metric: 'summary.json:loss', + param: 'params.yaml:dropout', + type: CustomPlotType.METRIC_VS_PARAM + } + ]) + + const mockSetCustomPlotsOrder = stub(plotsModel, 'setCustomPlotsOrder') + mockSetCustomPlotsOrder.returns(undefined) + + const mockSendTelemetryEvent = stub(Telemetry, 'sendTelemetryEvent') + const mockMessageReceived = getMessageReceivedEmitter(webview) + + mockMessageReceived.fire({ + type: MessageFromWebviewType.REMOVE_CUSTOM_PLOTS + }) + + await quickPickEvent + + expect(mockSetCustomPlotsOrder).to.not.be.called + expect(mockSendTelemetryEvent).to.not.be.called + }) }) }) From 4032f1c6460f9a4500dfc177a8f13c73712e7d4e Mon Sep 17 00:00:00 2001 From: julieg18 Date: Mon, 20 Mar 2023 12:30:15 -0500 Subject: [PATCH 40/40] Add testing for `name || label` code --- extension/src/test/fixtures/expShow/base/customPlots.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/extension/src/test/fixtures/expShow/base/customPlots.ts b/extension/src/test/fixtures/expShow/base/customPlots.ts index 37536b2a2f..3eb732d19f 100644 --- a/extension/src/test/fixtures/expShow/base/customPlots.ts +++ b/extension/src/test/fixtures/expShow/base/customPlots.ts @@ -37,26 +37,24 @@ export const customPlotsOrderFixture: CustomPlotsOrderValue[] = [ export const experimentsWithCheckpoints: ExperimentWithCheckpoints[] = [ { id: '12345', - label: '123', metrics: { 'summary.json': { accuracy: 0.3724166750907898, loss: 2.0205044746398926 } }, - name: 'exp-e7a67', + label: 'exp-e7a67', params: { 'params.yaml': { dropout: 0.15, epochs: 2 } }, checkpoints: [ { id: '12345', - label: '123', metrics: { 'summary.json': { accuracy: 0.3724166750907898, loss: 2.0205044746398926 } }, - name: 'exp-e7a67', + label: 'exp-e7a67', params: { 'params.yaml': { dropout: 0.15, epochs: 2 } } }, {