diff --git a/packages/safe-ds-eda/src/App.svelte b/packages/safe-ds-eda/src/App.svelte index 0ab3960fb..59541c42e 100644 --- a/packages/safe-ds-eda/src/App.svelte +++ b/packages/safe-ds-eda/src/App.svelte @@ -37,9 +37,9 @@
-
+
-
+
@@ -68,7 +68,6 @@ .tableWrapper { flex: 1; - overflow: scroll; } .resizer { diff --git a/packages/safe-ds-eda/src/components/TableView.svelte b/packages/safe-ds-eda/src/components/TableView.svelte index ba48c01a7..b2254c75f 100644 --- a/packages/safe-ds-eda/src/components/TableView.svelte +++ b/packages/safe-ds-eda/src/components/TableView.svelte @@ -3,55 +3,87 @@ import { throttle } from 'lodash'; import { currentState, preventClicks } from '../webviewState'; import CaretIcon from '../icons/Caret.svelte'; + import ErrorIcon from '../icons/Error.svelte'; + import FilterIcon from '../icons/Filter.svelte'; + import type { + Column, + PossibleColumnFilter, + Profiling, + ProfilingDetail, + ProfilingDetailStatistical, + } from '../../types/state'; + import ProfilingInfo from './profiling/ProfilingInfo.svelte'; + import { derived, writable, get } from 'svelte/store'; + import ColumnFilters from './columnFilters/ColumnFilters.svelte'; export let sidebarWidth: number; - $: if (sidebarWidth && tableContainer) { - updateTableSpace(); - } + let profilingImageWidth = 200; let showProfiling = false; let minTableWidth = 0; let numRows = 0; - const borderColumnWidth = 45; // Set in CSS, change here if changes in css const headerElements: HTMLElement[] = []; - const savedColumnWidths: Map = new Map(); + let maxProfilingItemCount = 0; + let savedColumnWidths = writable(new Map()); $: { if ($currentState.table) { minTableWidth = 0; numRows = 0; - $currentState.table.columns.forEach((column) => { + maxProfilingItemCount = 0; + $currentState.table.columns.forEach((column: [number, Column]) => { if (column[1].values.length > numRows) { numRows = column[1].values.length; } minTableWidth += 100; + + // Find which is the talles profiling type present in this table to adjust which profilings to give small height to, to have them adhere to good spacing + // (cannot give to tallest one, as then it will all be small) + if (column[1].profiling) { + let profilingItemCount = 0; + if (column[1].profiling.validRatio) profilingItemCount += 1; + if (column[1].profiling.missingRatio) profilingItemCount += 1; + for (const profilingItem of column[1].profiling.other) { + profilingItemCount += calcProfilingItemValue(profilingItem); + } + if (profilingItemCount > maxProfilingItemCount) { + maxProfilingItemCount = profilingItemCount; + } + } }); } } - const getColumnWidth = function (columnName: string): number { - if (savedColumnWidths.has(columnName)) { - return savedColumnWidths.get(columnName)!; + $: if (headerElements.length > 0) { + // Is svelte reactive but so far only runs once which is what we want, consideration to have loop in onMount that waits until headerElements is filled and then runs this code once + for (const column of headerElements) { + const columnName = column.innerText.trim(); + if (get(savedColumnWidths).has(columnName)) continue; // Only set intital width if not already set + + // // Example of how to calculate width based on content length, might be needed in place of profilingImageWidth approach again + // const baseWidth = 35; // Minimum width + // const scale = 55; + // // Use the logarithm of the character count, and scale it + // const width = baseWidth + Math.log(columnName.length + 1) * scale; + + const width = profilingImageWidth + 2 * 12; // Image width + 2 borders + + // Save the width for future use + savedColumnWidths.update((map) => { + map.set(columnName, width); + return map; + }); } - const baseWidth = 35; // Minimum width - const scale = 55; - - // Use the logarithm of the character count, and scale it - const width = baseWidth + Math.log(columnName.length + 1) * scale; - // Save the width for future use - savedColumnWidths.set(columnName, width); - - return width; - }; - - const getColumnWidthFreshNumber = function (columnName: string): number { - const baseWidth = 35; // Minimum width - const scale = 55; + lastHeight = tableContainer.clientHeight; // For recalculateVisibleRowCount + } - // Use the logarithm of the character count, and scale it - return baseWidth + Math.log(columnName.length + 1) * scale; + const handleMainCellClick = function (): void { + if (!$preventClicks) { + selectedColumnIndexes = []; + selectedRowIndexes = []; + } }; // --- Column resizing --- @@ -66,10 +98,12 @@ const currentWidth = startWidth + event.clientX - startX; requestAnimationFrame(() => { targetColumn.style.width = `${currentWidth}px`; - savedColumnWidths.set(targetColumn.innerText, currentWidth); + savedColumnWidths.update((map) => { + map.set(targetColumn.innerText.trim(), currentWidth); + return map; + }); + resizeWidthMap.set(targetColumn.innerText.trim(), currentWidth); }); - resizeWidthMap.set(targetColumn.innerText, currentWidth); - updateTableSpace(); } }; @@ -77,7 +111,6 @@ const startResizeDrag = function (event: MouseEvent, columnIndex: number): void { event.stopPropagation(); - clickOnColumn = true; const columnElement = headerElements[columnIndex]; isResizeDragging = true; startX = event.clientX; @@ -97,21 +130,20 @@ let isReorderDragging = false; let dragStartIndex: number | null = null; let dragCurrentIndex: number | null = null; - let draggedColumn: HTMLElement | null = null; + let draggedColumnName: string | null = null; + let reorderPrototype: HTMLElement; let savedColumnWidthBeforeReorder = 0; - let preventResizeTableSpaceUpdate = false; let holdTimeout: NodeJS.Timeout; let isClick = true; // Flag to distinguish between click and hold - let clickOnColumn = false; // For global window click clear of selection let currentMouseUpHandler: ((event: MouseEvent) => void) | null = null; // For being able to properly remove the mouseup listener when col clicked and not held const handleReorderDragOver = function (event: MouseEvent, columnIndex: number): void { - if (isReorderDragging && dragStartIndex !== null && draggedColumn) { + if (isReorderDragging && dragStartIndex !== null && draggedColumnName) { dragCurrentIndex = columnIndex; requestAnimationFrame(() => { - draggedColumn!.style.left = event.clientX + tableContainer.scrollLeft - sidebarWidth + 'px'; - draggedColumn!.style.top = event.clientY + 'px'; + reorderPrototype!.style.left = event.clientX + tableContainer.scrollLeft - sidebarWidth + 'px'; + reorderPrototype!.style.top = event.clientY + scrollTop + 'px'; }); } }; @@ -122,8 +154,6 @@ // Check if the left or right mouse button was pressed if (event.button !== 0 && event.button !== 2) return; - clickOnColumn = true; // For global window click clear of selection - if (event.button === 2) { // Right click handleColumnRightClick(event, columnIndex); @@ -138,17 +168,14 @@ isClick = true; // Assume it's a click initially holdTimeout = setTimeout(() => { + // Reorder drag start isClick = false; // If timeout completes, it's a hold document.addEventListener('mouseup', handleReorderDragEnd); - savedColumnWidthBeforeReorder = savedColumnWidths.get(headerElements[columnIndex].innerText)!; - preventResizeTableSpaceUpdate = true; // To not add the new space to current dragged column + savedColumnWidthBeforeReorder = get(savedColumnWidths).get(headerElements[columnIndex].innerText.trim())!; + draggedColumnName = headerElements[columnIndex].innerText.trim(); isReorderDragging = true; dragStartIndex = columnIndex; dragCurrentIndex = columnIndex; - draggedColumn = headerElements[columnIndex]; - draggedColumn.classList.add('dragging'); - savedColumnWidths.set(draggedColumn.innerText, 0); - updateTableSpace(); selectedColumnIndexes = []; // Clear so reordering doesn't interfere with selection }, 300); // milliseconds delay for hold detection @@ -172,13 +199,12 @@ const handleReorderDragEnd = function (): void { if (isReorderDragging && dragStartIndex !== null && dragCurrentIndex !== null) { - preventResizeTableSpaceUpdate = false; - if (draggedColumn) { - savedColumnWidths.set(draggedColumn.innerText, savedColumnWidthBeforeReorder); - draggedColumn.style.left = ''; - draggedColumn.style.top = ''; - draggedColumn.classList.remove('dragging'); - draggedColumn = null; + if (draggedColumnName) { + savedColumnWidths.update((map) => { + map.set(draggedColumnName!, savedColumnWidthBeforeReorder); + return map; + }); + draggedColumnName = null; } // Reset the z-index of all headers headerElements.forEach((header) => { @@ -201,8 +227,6 @@ isReorderDragging = false; dragStartIndex = null; dragCurrentIndex = null; - updateTableSpace(); - updateTableSpace(); // Have to somehow call twice, first time it thinks the window is around 10px bigger than it is } }; @@ -228,18 +252,32 @@ addColumnToSelection(columnIndex); } } else { - // Replace the current selection - // Replace the current selection with a new array to trigger reactivity - setSelectionToColumn(columnIndex); + const index = selectedColumnIndexes.indexOf(columnIndex); + if (index > -1 && selectedColumnIndexes.length === 1) { + // Already selected, so clear selection + selectedColumnIndexes = []; + } else { + // Not selected, replace the current selection + // Replace the current selection with a new array to trigger reactivity + setSelectionToColumn(columnIndex); + } } }; const addColumnToSelection = function (columnIndex: number): void { + if (selectedRowIndexes.length > 0) { + selectedRowIndexes = []; + } + // Add the index and create a new array to trigger reactivity selectedColumnIndexes = [...selectedColumnIndexes, columnIndex]; }; const removeColumnFromSelection = function (columnIndex: number, selectedColumnIndexesIndex?: number): void { + if (selectedRowIndexes.length > 0) { + selectedRowIndexes = []; + } + // Remove the index and create a new array to trigger reactivity selectedColumnIndexes = [ ...selectedColumnIndexes.slice(0, selectedColumnIndexesIndex ?? selectedColumnIndexes.indexOf(columnIndex)), @@ -250,13 +288,16 @@ }; const setSelectionToColumn = function (columnIndex: number): void { + if (selectedRowIndexes.length > 0) { + selectedRowIndexes = []; + } + // Replace the current selection with a new array to trigger reactivity selectedColumnIndexes = [columnIndex]; }; // --- Row selecting --- let selectedRowIndexes: number[] = []; - let clickOnRow = false; const handleRowClick = function (event: MouseEvent, rowIndex: number): void { // Logic for what happens when a row is clicked @@ -264,7 +305,9 @@ return; } - clickOnRow = true; // For global window click clear of selection + if (selectedColumnIndexes.length > 0) { + selectedColumnIndexes = []; + } // Check if Ctrl (or Cmd on Mac) is held down if (event.ctrlKey || event.metaKey) { @@ -280,16 +323,22 @@ selectedRowIndexes = [...selectedRowIndexes, rowIndex]; } } else { - // Replace the current selection - // Replace the current selection with a new array to trigger reactivity - selectedRowIndexes = [rowIndex]; + const index = selectedRowIndexes.indexOf(rowIndex); + if (index > -1 && selectedRowIndexes.length === 1) { + // Already selected, so clear selection + selectedRowIndexes = []; + } else { + // Not selected, replace the current selection + // Replace the current selection with a new array to trigger reactivity + selectedRowIndexes = [rowIndex]; + } } }; // --- Scroll loading --- let tableContainer: HTMLElement; // Reference to the table container const rowHeight = 33; // Adjust based on your row height - const buffer = 25; // Number of rows to render outside the viewport + const buffer = 40; // Number of rows to render outside the viewport let visibleStart = 0; let visibleEnd = 0; let visibleRowCount = 10; @@ -298,7 +347,7 @@ const updateVisibleRows = function (): void { visibleStart = Math.max(0, Math.floor(scrollTop / rowHeight) - buffer); - visibleEnd = visibleStart + visibleRowCount; + visibleEnd = visibleStart + visibleRowCount + buffer; }; const throttledUpdateVisibleRows = throttle(updateVisibleRows, 40); @@ -322,72 +371,13 @@ const throttledRecalculateVisibleRowCount = throttle(recalculateVisibleRowCount, 20); - // --- Min Table with --- - const throttledUpdateTableSpace = throttle(() => { - if (!preventResizeTableSpaceUpdate) { - updateTableSpace(); - } - }, 100); - - const updateTableSpace = function (): void { - const newPossibleSpace = tableContainer.clientWidth; - - const utilitySpace = borderColumnWidth * 2; // 2 border columns - let beforeWidth = utilitySpace; - for (const width of savedColumnWidths.values()) { - if (width === 0) { - } - beforeWidth += width; - } - - if (newPossibleSpace > beforeWidth) { - // Extend all column widths proportionally with new space - for (const column of headerElements) { - const newWidth = column.offsetWidth + (newPossibleSpace - beforeWidth) / headerElements.length; - column.style.width = newWidth + 'px'; - savedColumnWidths.set(column.innerText, newWidth); - } - } else { - // Shrink all column widths proportionally with new space if not below minimum width dedicated by a: width by header text or b: with by manual resize - for (const column of headerElements) { - const newWidth = column.offsetWidth - (beforeWidth - newPossibleSpace) / headerElements.length; - if (resizeWidthMap.has(column.innerText)) { - // User resized manually, so don't shrink below that - if (resizeWidthMap.get(column.innerText)! <= newWidth) { - column.style.width = newWidth + 'px'; - savedColumnWidths.set(column.innerText, newWidth); - } else if (column.offsetWidth !== resizeWidthMap.get(column.innerText)!) { - // To update even on fast resize - column.style.width = resizeWidthMap.get(column.innerText)! + 'px'; - savedColumnWidths.set(column.innerText, resizeWidthMap.get(column.innerText)!); - } - } else { - // Use the minimum width dedicated by the header text - const minWidth = getColumnWidthFreshNumber(column.innerText); - if (minWidth <= newWidth) { - column.style.width = newWidth + 'px'; - savedColumnWidths.set(column.innerText, newWidth); - } else if (column.clientWidth !== minWidth) { - // To update even on fast resize - column.style.width = minWidth + 'px'; - savedColumnWidths.set(column.innerText, minWidth); - } - } - } - } - }; - - $: if (headerElements.length > 0) { - // Is svelte reactive but so far only runs once which is what we want, consideration to have loop in onMount that waits until headerElements is filled and then runs this code once - lastHeight = tableContainer.clientHeight; - updateTableSpace(); - } - // --- Right clicks --- + let currentContextMenu: HTMLElement | null = null; + + // Column header right click let showingColumnHeaderRightClickMenu = false; let rightClickedColumnIndex = -1; let rightClickColumnMenuElement: HTMLElement; - let currentContextMenu: HTMLElement | null = null; const handleColumnRightClick = function (event: MouseEvent, columnIndex: number): void { // Logic for what happens when a header is right clicked @@ -401,25 +391,83 @@ rightClickColumnMenuElement!.style.top = event.clientY + scrollTop + 'px'; }); - // Click anywhere else to close the menu, context menu selection has to prevent propagation + // Click anywhere else to close the menu window.addEventListener('click', handleRightClickEnd); }; + // Filter context menu + let showingFilterContextMenu = false; + let filterColumnIndex = -1; + let filterContextMenuElement: HTMLElement; + + const handleFilterContextMenu = function (event: MouseEvent, columnIndex: number): void { + if (event.button !== 0 || $preventClicks) return; + + // Logic for what happens when a filter icon is clicked + event.stopPropagation(); + doDefaultContextMenuSetup(); + showingFilterContextMenu = true; + filterColumnIndex = columnIndex; + + requestAnimationFrame(() => { + currentContextMenu = filterContextMenuElement; // So scrolling can edit the position, somehow assignment does only work in requestAnimationFrame, maybe bc of delay, could lead to bugs maybe in future, keep note of + filterContextMenuElement!.style.left = event.clientX + tableContainer.scrollLeft - sidebarWidth + 'px'; + filterContextMenuElement!.style.top = event.clientY + scrollTop + 'px'; + }); + + // Click anywhere else to close the menu, if not clicked in the menu + window.addEventListener('mousedown', handleRightClickEnd); + }; + + // Scaling methods + const doDefaultContextMenuSetup = function (): void { preventClicks.set(true); disableNonContextMenuEffects(); }; - const handleRightClickEnd = function (): void { + const handleRightClickEnd = function (event: MouseEvent): void { + const generalCleanup = function (): void { + restoreNonContextMenuEffects(); + setTimeout(() => preventClicks.set(false), 100); // To give time for relevant click events to be prevented + currentContextMenu = null; + window.removeEventListener('click', handleRightClickEnd); + window.removeEventListener('mousedown', handleRightClickEnd); + }; + // Code specific to each menu - showingColumnHeaderRightClickMenu = false; - rightClickedColumnIndex = -1; - // ---- - - restoreNonContextMenuEffects(); - preventClicks.set(false); - currentContextMenu = null; - window.removeEventListener('click', handleRightClickEnd); + if (showingColumnHeaderRightClickMenu) { + showingColumnHeaderRightClickMenu = false; + rightClickedColumnIndex = -1; + generalCleanup(); + } + if (showingFilterContextMenu) { + if (event.target instanceof HTMLElement) { + let element = event.target; + + const hasParentWithClass = (elementToScan: HTMLElement, className: string) => { + let currentElement: HTMLElement = elementToScan; + while (currentElement && currentElement !== document.body) { + if (currentElement.classList.contains(className)) { + return true; + } + if (!currentElement.parentElement) { + return false; + } + currentElement = currentElement.parentElement; + } + return false; + }; + + // Check if the clicked element or any of its parents have the 'contextMenu' class + if (hasParentWithClass(element, 'contextMenu')) { + return; + } + } + showingFilterContextMenu = false; + filterColumnIndex = -1; + generalCleanup(); + } }; const originalHoverStyles = new Map(); @@ -471,65 +519,132 @@ }; // --- Profiling --- + let fullHeadBackground: HTMLElement; + let profilingInfo: HTMLElement; + const toggleProfiling = function (): void { - if (!$preventClicks) showProfiling = !showProfiling; + if (!$preventClicks) { + showProfiling = !showProfiling; + if (showProfiling) { + setTimeout(() => { + fullHeadBackground.style.height = 2 * rowHeight + profilingInfo.clientHeight + 'px'; + }, 700); // 700ms is the transition time of the profiling info opening/incresing in height + } else { + fullHeadBackground.style.height = rowHeight * 2 + 'px'; + } + } }; - // --- Lifecycle --- - let interval: NodeJS.Timeout; + const getOptionalProfilingHeight = function (profiling: Profiling): string { + let profilingItemCount = 0; - const clearSelections = function (event: MouseEvent): void { - // Clears selections if last click was not on a column or row and currrent click is not on a context menu item if context menu is open - // WARN/TODO: Does not yet work for subemnus in context menus or menus with non possible closing clicks, those will need yet another class to be detected and handled - // This also prepares selection clearing for next iteration if click was on column or row - - // Clear column selection if approriate - if ( - !clickOnColumn && - !( - currentContextMenu && - event.target instanceof HTMLElement && - !(event.target as HTMLElement).classList.contains('contextItem') - ) - ) { - // Clear if click last item clicked was not on a column or if current click is on a context menu item if context menu is open, - // which should just close the context menu and not clear selection - selectedColumnIndexes = []; + if (profiling.validRatio) profilingItemCount += 1; + if (profiling.missingRatio) profilingItemCount += 1; + + for (const profilingItem of profiling.other) { + profilingItemCount += calcProfilingItemValue(profilingItem); } - clickOnColumn = false; // meaning if next click is not on a column, selection will be cleared in next iteration - - // Clear row selection if approriate - if ( - !clickOnRow && - !( - currentContextMenu && - event.target instanceof HTMLElement && - !(event.target as HTMLElement).classList.contains('contextItem') - ) - ) { - // Clear if click last item clicked was not on a row or if current click is on a context menu item if context menu is open, - // which should just close the context menu and not clear selection - selectedRowIndexes = []; + + if (profilingItemCount === maxProfilingItemCount) { + return ''; + } else { + return '30px'; + } + }; + + const calcProfilingItemValue = function (profilingItem: ProfilingDetail): number { + // To edit when Profiling type scales/changes + if (profilingItem.type === 'image') { + return 9; // Bigger than normal text line, should be set to 3x line height + } else { + return 1; + } + }; + + // As store to update on profiling changes + const hasProfilingErrors = derived(currentState, ($currentState) => { + if (!$currentState.table) return false; + for (const column of $currentState.table!.columns) { + if (!column[1].profiling) return false; + if ( + column[1].profiling.missingRatio?.interpretation === 'error' || + column[1].profiling.validRatio?.interpretation === 'error' + ) { + return true; + } + for (const profilingItem of column[1].profiling.other) { + if (profilingItem.type === 'numerical' && profilingItem.interpretation === 'error') { + return true; + } + } + } + return false; + }); + + const getPosiibleColumnFilters = function (columnIndex: number): PossibleColumnFilter[] { + if (!$currentState.table) return []; + + const column = $currentState.table.columns[columnIndex][1]; + + if (!column.profiling) return []; + + const possibleColumnFilters: PossibleColumnFilter[] = []; + + if (column.type === 'categorical') { + const profilingCategories: ProfilingDetailStatistical[] = column.profiling.other.filter( + (profilingItem: ProfilingDetail) => + profilingItem.type === 'numerical' && profilingItem.interpretation === 'category', + ) as ProfilingDetailStatistical[]; + + // If there is distinct categories in profiling, use those as filter options, else use search string + if (profilingCategories.length > 0) { + possibleColumnFilters.push({ + type: 'specificValue', + values: ['-'].concat(profilingCategories.map((profilingItem) => profilingItem.name)), + columnName: column.name, + }); + } else { + possibleColumnFilters.push({ + type: 'searchString', + columnName: column.name, + }); + } + } else { + const colMax = column.values.reduce( + (acc: number, val: number) => Math.max(acc, val), + Number.NEGATIVE_INFINITY, + ); + const colMin = column.values.reduce( + (acc: number, val: number) => Math.min(acc, val), + Number.NEGATIVE_INFINITY, + ); + + possibleColumnFilters.push({ + type: 'valueRange', + min: colMin, + max: colMax, + columnName: column.name, + }); } - clickOnRow = false; // meaning if next click is not on a row, selection will be cleared in next iteration + + return possibleColumnFilters; }; + // --- Lifecycle --- + let interval: NodeJS.Timeout; + onMount(() => { updateScrollTop(); recalculateVisibleRowCount(); tableContainer.addEventListener('scroll', throttledUpdateVisibleRows); tableContainer.addEventListener('scroll', updateScrollTop); window.addEventListener('resize', throttledRecalculateVisibleRowCount); - window.addEventListener('resize', throttledUpdateTableSpace); - window.addEventListener('click', clearSelections); interval = setInterval(updateVisibleRows, 500); // To catch cases of fast scroll bar scrolling that leave table blank return () => { tableContainer.removeEventListener('scroll', throttledUpdateVisibleRows); tableContainer.addEventListener('scroll', updateScrollTop); window.removeEventListener('resize', throttledRecalculateVisibleRowCount); - window.removeEventListener('resize', throttledUpdateTableSpace); - window.removeEventListener('click', clearSelections); clearInterval(interval); }; }); @@ -539,10 +654,17 @@ {#if !$currentState.table} Loading ... {:else} -
+
+
+ - + handleColumnInteractionStart(event, index)} on:mousemove={(event) => throttledHandleReorderDragOver(event, index)} >{column[1].name}
startResizeDrag(event, index)} - >
+ class="filterIconWrapper" + on:mousedown={(event) => handleFilterContextMenu(event, index)} + > + + +
+
+ +
+
+ +
+
+ +
startResizeDrag(event, index)}>
{/each} - + + - {#each $currentState.table.columns as _column, index} + {#each $currentState.table.columns as column, index} {/each} @@ -595,9 +739,10 @@ throttledHandleReorderDragOver(event, $currentState.table?.columns.length ?? 0)} > - + + - {#each $currentState.table.columns as _column, i} - + {#each $currentState.table.columns as _column, index} + {#if index !== $currentState.table.columns.length - 1} + + {/if} {/each} + {#each Array(Math.min(visibleEnd, numRows) - visibleStart) as _, i} - + {column[1].values[visibleStart + i] !== null && + column[1].values[visibleStart + i] !== undefined + ? column[1].values[visibleStart + i] + : ''} {/each} + {/if} {#if showingColumnHeaderRightClickMenu}
@@ -687,6 +855,11 @@ {/if}
{/if} + {#if showingFilterContextMenu} +
+ +
+ {/if} diff --git a/packages/safe-ds-eda/src/components/columnFilters/ColumnFilters.svelte b/packages/safe-ds-eda/src/components/columnFilters/ColumnFilters.svelte new file mode 100644 index 000000000..50a0cc14d --- /dev/null +++ b/packages/safe-ds-eda/src/components/columnFilters/ColumnFilters.svelte @@ -0,0 +1,42 @@ + + +
+ {#each possibleFilters as filter} + {#if filter.type === 'specificValue'} +
+ Filter by value + +
+ {/if} + {/each} +
+ + diff --git a/packages/safe-ds-eda/src/components/profiling/ProfilingInfo.svelte b/packages/safe-ds-eda/src/components/profiling/ProfilingInfo.svelte new file mode 100644 index 000000000..d4d3b4056 --- /dev/null +++ b/packages/safe-ds-eda/src/components/profiling/ProfilingInfo.svelte @@ -0,0 +1,100 @@ + + +
+
+
+ {profiling.validRatio.name}: + {profiling.validRatio.value} +
+
+ {profiling.missingRatio.name}: + {profiling.missingRatio.value} +
+
+
+ {#each profiling.other as profilingItem} + {#if profilingItem.type === 'text'} +
+ {profilingItem.value} +
+ {:else if profilingItem.type === 'numerical'} +
+ {profilingItem.name}: + {profilingItem.value} +
+ {:else if profilingItem.type === 'image'} +
+ profiling plot +
+ {/if} + {/each} +
+
+ + diff --git a/packages/safe-ds-eda/src/icons/BarPlot.svelte b/packages/safe-ds-eda/src/icons/BarPlot.svelte index fed3904b6..f3728e13f 100644 --- a/packages/safe-ds-eda/src/icons/BarPlot.svelte +++ b/packages/safe-ds-eda/src/icons/BarPlot.svelte @@ -1,6 +1,6 @@ diff --git a/packages/safe-ds-eda/src/icons/Caret.svelte b/packages/safe-ds-eda/src/icons/Caret.svelte index 7ffe2c10e..4046bb43f 100644 --- a/packages/safe-ds-eda/src/icons/Caret.svelte +++ b/packages/safe-ds-eda/src/icons/Caret.svelte @@ -1,3 +1,7 @@ + + - + diff --git a/packages/safe-ds-eda/src/icons/Error.svelte b/packages/safe-ds-eda/src/icons/Error.svelte new file mode 100644 index 000000000..25dcad6b9 --- /dev/null +++ b/packages/safe-ds-eda/src/icons/Error.svelte @@ -0,0 +1,6 @@ + diff --git a/packages/safe-ds-eda/src/icons/Filter.svelte b/packages/safe-ds-eda/src/icons/Filter.svelte new file mode 100644 index 000000000..6556842d7 --- /dev/null +++ b/packages/safe-ds-eda/src/icons/Filter.svelte @@ -0,0 +1,10 @@ + + + diff --git a/packages/safe-ds-eda/src/icons/LinePlot.svelte b/packages/safe-ds-eda/src/icons/LinePlot.svelte index 31ee95841..7bf240b0d 100644 --- a/packages/safe-ds-eda/src/icons/LinePlot.svelte +++ b/packages/safe-ds-eda/src/icons/LinePlot.svelte @@ -1,6 +1,6 @@ diff --git a/packages/safe-ds-eda/src/icons/Table.svelte b/packages/safe-ds-eda/src/icons/Table.svelte index d3b185d04..3ff160b6a 100644 --- a/packages/safe-ds-eda/src/icons/Table.svelte +++ b/packages/safe-ds-eda/src/icons/Table.svelte @@ -1,6 +1,6 @@ diff --git a/packages/safe-ds-eda/src/icons/Undo.svelte b/packages/safe-ds-eda/src/icons/Undo.svelte index 0642d4590..f0f87726a 100644 --- a/packages/safe-ds-eda/src/icons/Undo.svelte +++ b/packages/safe-ds-eda/src/icons/Undo.svelte @@ -1,6 +1,6 @@ diff --git a/packages/safe-ds-eda/src/webviewState.ts b/packages/safe-ds-eda/src/webviewState.ts index 0e51f3d8a..a19d1459d 100644 --- a/packages/safe-ds-eda/src/webviewState.ts +++ b/packages/safe-ds-eda/src/webviewState.ts @@ -1,14 +1,14 @@ import type { FromExtensionMessage } from '../types/messaging'; import type { State } from '../types/state'; import * as extensionApi from './apis/extensionApi'; -import { writable } from 'svelte/store'; +import { get, writable } from 'svelte/store'; let currentTabIndex = writable(0); let preventClicks = writable(false); // Define the stores, current state to default in case the extension never calls setWebviewState( Shouldn't happen) -let currentState = writable({ tableIdentifier: window.tableIdentifier, history: [], defaultState: true }); +let currentState = writable({ tableIdentifier: undefined, history: [], defaultState: true }); // Set Global states whenever updatedAllStates changes currentState.subscribe(($currentState) => { @@ -26,6 +26,32 @@ window.addEventListener('message', (event) => { // This should be fired immediately whenever the panel is created or made visible again currentState.set(message.value); break; + case 'setProfiling': + if (get(currentState) && get(currentState).table) { + currentState.update((state) => { + return { + ...state, + table: { + ...state.table!, + columns: state.table!.columns.map((column) => { + const profiling = message.value.find((p) => p.columnName === column[1].name); + if (profiling) { + return [ + column[0], + { + ...column[1], + profiling: profiling.profiling, + }, + ]; + } else { + return column; + } + }), + }, + }; + }); + } + break; } }); diff --git a/packages/safe-ds-eda/types/globals.d.ts b/packages/safe-ds-eda/types/globals.d.ts index 881c89e61..d78d1ed9d 100644 --- a/packages/safe-ds-eda/types/globals.d.ts +++ b/packages/safe-ds-eda/types/globals.d.ts @@ -5,6 +5,5 @@ declare global { injVscode: { postMessage: (message: ToExtensionMessage) => void; }; - tableIdentifier: string; } } diff --git a/packages/safe-ds-eda/types/messaging.d.ts b/packages/safe-ds-eda/types/messaging.d.ts index 2b0a645f8..438c7081e 100644 --- a/packages/safe-ds-eda/types/messaging.d.ts +++ b/packages/safe-ds-eda/types/messaging.d.ts @@ -46,4 +46,9 @@ interface FromExtensionSetStateMessage extends FromExtensionCommandMessage { value: defaultTypes.State; } -export type FromExtensionMessage = FromExtensionSetStateMessage; +interface FromExtensionSetProfilingMessage extends FromExtensionCommandMessage { + command: 'setProfiling'; + value: { columnName: string; profiling: defaultTypes.Profiling }[]; +} + +export type FromExtensionMessage = FromExtensionSetStateMessage | FromExtensionSetProfilingMessage; diff --git a/packages/safe-ds-eda/types/state.d.ts b/packages/safe-ds-eda/types/state.d.ts index 26080d7a8..4c42b8907 100644 --- a/packages/safe-ds-eda/types/state.d.ts +++ b/packages/safe-ds-eda/types/state.d.ts @@ -71,32 +71,35 @@ export interface Table { // ------------ Types for the Profiling ----------- export interface Profiling { - top: ProfilingDetail[]; - bottom: ProfilingDetail[]; + validRatio: ProfilingDetailStatistical; + missingRatio: ProfilingDetailStatistical; + other: ProfilingDetail[]; } export interface ProfilingDetailBase { type: 'numerical' | 'image' | 'name'; - name: string; + value: string; +} + +interface ProfilingDetailText extends ProfilingDetailBase { + interpretation: 'warn' | 'error' | 'default' | 'important' | 'good'; } -export interface ProfilingDetailStatistical extends ProfilingDetailBase { +export interface ProfilingDetailStatistical extends ProfilingDetailText { type: 'numerical'; name: string; - value: number; - color?: string; + value: string; + interpretation: ProfilingDetailText['interpretation'] | 'category'; // 'category' needed for filters, to show distinct values } export interface ProfilingDetailImage extends ProfilingDetailBase { type: 'image'; - name: string; - encodedImage: string; + value: Base64Image; } -export interface ProfilingDetailName extends ProfilingDetailBase { - type: 'name'; - name: string; - color?: string; +export interface ProfilingDetailName extends ProfilingDetailText { + type: 'text'; + value: string; } export type ProfilingDetail = ProfilingDetailStatistical | ProfilingDetailImage | ProfilingDetailName; @@ -109,7 +112,7 @@ export interface ColumnBase { hidden: boolean; highlighted: boolean; appliedSort: 'asc' | 'desc' | null; - profiling: Profiling; + profiling?: Profiling; } export interface NumericalColumn extends ColumnBase { @@ -135,24 +138,39 @@ export interface ColumnFilterBase extends FilterBase { columnName: string; } -export interface SearchStringFilter extends ColumnFilterBase { +export interface PossibleSearchStringFilter extends ColumnFilterBase { type: 'searchString'; +} + +export interface SearchStringFilter extends PossibleSearchStringFilter { searchString: string; } -export interface ValueRangeFilter extends ColumnFilterBase { +export interface PossibleValueRangeFilter extends ColumnFilterBase { type: 'valueRange'; min: number; max: number; } +export interface ValueRangeFilter extends PossibleValueRangeFilter { + currentMin: number; + currentMax: number; +} + +export interface PossibleSpecificValueFilter extends ColumnFilterBase { + type: 'specificValue'; + values: string[]; +} + export interface SpecificValueFilter extends ColumnFilterBase { type: 'specificValue'; - value: number; + value: string; } -export type NumericalFilter = ValueRangeFilter | SpecificValueFilter; -export type CategoricalFilter = SearchStringFilter; +export type NumericalFilter = ValueRangeFilter; +export type CategoricalFilter = SearchStringFilter | SpecificValueFilter; + +export type PossibleColumnFilter = PossibleValueRangeFilter | PossibleSearchStringFilter | PossibleSpecificValueFilter; export interface TableFilter extends FilterBase { type: 'hideMissingValueColumns' | 'hideNonNumericalColumns' | 'hideDuplicateRows' | 'hideRowsWithOutliers'; @@ -179,3 +197,10 @@ export interface ProfilingSettings extends ProfilingSettingsBase { sum: boolean; variance: boolean; } + +// ------------ Types for general objects ----------- + +export interface Base64Image { + format: string; + bytes: string; +} diff --git a/packages/safe-ds-vscode/media/styles.css b/packages/safe-ds-vscode/media/styles.css index dcd555217..61b357140 100644 --- a/packages/safe-ds-vscode/media/styles.css +++ b/packages/safe-ds-vscode/media/styles.css @@ -1,12 +1,15 @@ :root { --primary-color: #036ed1; --primary-color-desaturated: rgb(3 109 209 / 16.4%); + --error-color: #a01417; + --warn-color: #a08b14; --bg-bright: white; --bg-dark: #f2f2f2; --bg-medium: #f9f9f9; --font-dark: #292929; --font-light: #6d6d6d; --font-bright: #fff; + --transparent: #ffffffa1; } .noSelect { diff --git a/packages/safe-ds-vscode/src/extension/eda/apis/runnerApi.ts b/packages/safe-ds-vscode/src/extension/eda/apis/runnerApi.ts new file mode 100644 index 000000000..f3c0a3fde --- /dev/null +++ b/packages/safe-ds-vscode/src/extension/eda/apis/runnerApi.ts @@ -0,0 +1,416 @@ +import { Base64Image, Column, Profiling, ProfilingDetailStatistical, Table } from '@safe-ds/eda/types/state.js'; +import { SafeDsServices, ast, messages } from '@safe-ds/lang'; +import { LangiumDocument, AstNode } from 'langium'; +import { printOutputMessage } from '../../output.ts'; +import * as vscode from 'vscode'; +import crypto from 'crypto'; +import { getPipelineDocument } from '../../mainClient.ts'; +import { CODEGEN_PREFIX } from '../../../../../safe-ds-lang/src/language/generation/safe-ds-python-generator.ts'; + +export class RunnerApi { + services: SafeDsServices; + pipelinePath: vscode.Uri; + pipelineName: string; + pipelineNode: ast.SdsPipeline; + baseDocument: LangiumDocument | undefined; + placeholderCounter = 0; + + constructor( + services: SafeDsServices, + pipelinePath: vscode.Uri, + pipelineName: string, + pipelineNode: ast.SdsPipeline, + ) { + this.services = services; + this.pipelinePath = pipelinePath; + this.pipelineName = pipelineName; + this.pipelineNode = pipelineNode; + getPipelineDocument(this.pipelinePath).then((doc) => { + // Get here to avoid issues because of chanigng file + // Make sure to create new instance of RunnerApi if pipeline execution of fresh pipeline is needed + // (e.g. launching of extension on table with existing state but no current panel) + this.baseDocument = doc; + }); + } + + private async addToAndExecutePipeline(pipelineExecutionId: string, addedLines: string): Promise { + return new Promise(async (resolve, reject) => { + if (!this.baseDocument) { + reject('Document not found'); + return; + } + + const documentText = this.baseDocument.textDocument.getText(); + + const endOfPipeline = this.pipelineNode.$cstNode?.end; + if (!endOfPipeline) { + reject('Pipeline not found'); + return; + } + + const beforePipelineEnd = documentText.substring(0, endOfPipeline - 1); + const afterPipelineEnd = documentText.substring(endOfPipeline - 1); + const newDocumentText = beforePipelineEnd + addedLines + afterPipelineEnd; + + const newDoc = this.services.shared.workspace.LangiumDocumentFactory.fromString( + newDocumentText, + this.pipelinePath, + ); + await this.services.runtime.Runner.executePipeline(pipelineExecutionId, newDoc, this.pipelineName); + + const runtimeCallback = (message: messages.RuntimeProgressMessage) => { + if (message.id !== pipelineExecutionId) { + return; + } + if (message.data === 'done') { + this.services.runtime.Runner.removeMessageCallback(runtimeCallback, 'runtime_progress'); + this.services.runtime.Runner.removeMessageCallback(errorCallback, 'runtime_error'); + resolve(); + } + }; + const errorCallback = (message: messages.RuntimeErrorMessage) => { + if (message.id !== pipelineExecutionId) { + return; + } + this.services.runtime.Runner.removeMessageCallback(runtimeCallback, 'runtime_progress'); + this.services.runtime.Runner.removeMessageCallback(errorCallback, 'runtime_error'); + reject(message.data); + }; + this.services.runtime.Runner.addMessageCallback(runtimeCallback, 'runtime_progress'); + this.services.runtime.Runner.addMessageCallback(errorCallback, 'runtime_error'); + + setTimeout(() => { + reject('Pipeline execution timed out'); + }, 3000000); + }); + } + + // --- SDS code generation --- + + private sdsStringForMissingValueRatioByColumnName( + columnName: string, + tablePlaceholder: string, + newPlaceholderName: string, + ): string { + return ( + 'val ' + + newPlaceholderName + + ' = ' + + tablePlaceholder + + '.getColumn("' + + columnName + + '").missingValueRatio(); \n' + ); + } + + private sdsStringForIDnessByColumnName(columnName: string, tablePlaceholder: string, newPlaceholderName: string) { + return 'val ' + newPlaceholderName + ' = ' + tablePlaceholder + '.getColumn("' + columnName + '").idness(); \n'; + } + + private sdsStringForHistogramByColumnName( + columnName: string, + tablePlaceholder: string, + newPlaceholderName: string, + ) { + return ( + 'val ' + + newPlaceholderName + + ' = ' + + tablePlaceholder + + '.getColumn("' + + columnName + + '").plotHistogram(); \n' + ); + } + + // --- Placeholder handling --- + + private genPlaceholderName(): string { + return CODEGEN_PREFIX + this.placeholderCounter++; + } + + private async getPlaceholderValue(placeholder: string, pipelineExecutionId: string): Promise { + return new Promise((resolve) => { + if (placeholder === '') { + resolve(undefined); + } + + const placeholderValueCallback = (message: messages.PlaceholderValueMessage) => { + if (message.id !== pipelineExecutionId || message.data.name !== placeholder) { + return; + } + this.services.runtime.Runner.removeMessageCallback(placeholderValueCallback, 'placeholder_value'); + resolve(message.data.value); + }; + + this.services.runtime.Runner.addMessageCallback(placeholderValueCallback, 'placeholder_value'); + printOutputMessage('Getting placeholder from Runner ...'); + this.services.runtime.Runner.sendMessageToPythonServer( + messages.createPlaceholderQueryMessage(pipelineExecutionId, placeholder), + ); + + setTimeout(() => { + resolve(undefined); + }, 30000); + }); + } + + // --- Public API --- + + public async getTableByPlaceholder(tableName: string, pipelineExecutionId: string): Promise
throttledHandleReorderDragOver(event, 0)}># throttledHandleReorderDragOver(event, $currentState.table?.columns.length ?? 0)}>#
throttledHandleReorderDragOver(event, 0)} > throttledHandleReorderDragOver(event, index)} >
- Heyyyyyyyyyyy
Hey
Hey
Hey
Hey
Hey
Hey + {#if !column[1].profiling} +
Loading ...
+ {:else} + + {/if}
throttledHandleReorderDragOver(event, 0)} >
{showProfiling ? 'Hide Profiling' : 'Show Profiling'} -
+ {#if $hasProfilingErrors} +
+ +
+ {/if} +
throttledHandleReorderDragOver(event, i + 1)} - > - throttledHandleReorderDragOver(event, index + 1)} + > +
throttledHandleReorderDragOver(event, 0)} @@ -639,10 +793,15 @@ > {#each $currentState.table.columns as column, index} throttledHandleReorderDragOver(event, index)} class:selectedColumn={selectedColumnIndexes.includes(index) || selectedRowIndexes.includes(visibleStart + i)} - >{column[1].values[visibleStart + i] || ''} No data {/if} + {#if draggedColumnName} + {draggedColumnName} +
{ + const pythonTableColumns = await this.getPlaceholderValue(tableName, pipelineExecutionId); + if (pythonTableColumns) { + const table: Table = { + totalRows: 0, + name: tableName, + columns: [] as Table['columns'], + appliedFilters: [] as Table['appliedFilters'], + }; + + let i = 0; + let currentMax = 0; + for (const [columnName, columnValues] of Object.entries(pythonTableColumns)) { + if (!Array.isArray(columnValues)) { + continue; + } + if (currentMax < columnValues.length) { + currentMax = columnValues.length; + } + + const isNumerical = typeof columnValues[0] === 'number'; + const columnType = isNumerical ? 'numerical' : 'categorical'; + + const column: Column = { + name: columnName, + values: columnValues, + type: columnType, + hidden: false, + highlighted: false, + appliedFilters: [], + appliedSort: null, + coloredHighLow: false, + }; + table.columns.push([i++, column]); + } + table.totalRows = currentMax; + table.visibleRows = currentMax; + + return table; + } else { + return undefined; + } + } + + public async getProfiling(table: Table): Promise<{ columnName: string; profiling: Profiling }[]> { + const columns = table.columns; + + let sdsStrings = ''; + + const columnNameToPlaceholderMVNameMap = new Map(); // Mapping random placeholder name for missing value ratio back to column name + const missingValueRatioMap = new Map(); // Saved by random placeholder name + + const columnNameToPlaceholderIDnessNameMap = new Map(); // Mapping random placeholder name for IDness back to column name + const idnessMap = new Map(); // Saved by random placeholder name + + const columnNameToPlaceholderHistogramNameMap = new Map(); // Mapping random placeholder name for histogram back to column name + const histogramMap = new Map(); // Saved by random placeholder name + + // Generate SDS code to get missing value ratio for each column + outer: for (const column of columns) { + const newMvPlaceholderName = this.genPlaceholderName(); + columnNameToPlaceholderMVNameMap.set(column[1].name, newMvPlaceholderName); + sdsStrings += this.sdsStringForMissingValueRatioByColumnName( + column[1].name, + table.name, + newMvPlaceholderName, + ); + + // Only need to check IDness for non-numerical columns + if (column[1].type !== 'numerical') { + const newIDnessPlaceholderName = this.genPlaceholderName(); + columnNameToPlaceholderIDnessNameMap.set(column[1].name, newIDnessPlaceholderName); + sdsStrings += this.sdsStringForIDnessByColumnName(column[1].name, table.name, newIDnessPlaceholderName); + + // Find unique values + // TODO reevaluate when image stuck problem fixed + let uniqueValues = new Set(); + for (let j = 0; j < column[1].values.length; j++) { + uniqueValues.add(column[1].values[j]); + if (uniqueValues.size > 10) { + continue outer; + } + } + if (uniqueValues.size <= 3 || uniqueValues.size > 10) { + // Must match conidtions below that choose to display histogram + continue; // This historam only generated if between 4-10 categorigal uniques or numerical type + } + } + + // Histogram for numerical columns or categorical columns with 4-10 unique values + const newHistogramPlaceholderName = this.genPlaceholderName(); + columnNameToPlaceholderHistogramNameMap.set(column[1].name, newHistogramPlaceholderName); + sdsStrings += this.sdsStringForHistogramByColumnName( + column[1].name, + table.name, + newHistogramPlaceholderName, + ); + } + + // Execute with generated SDS code + const pipelineExecutionId = crypto.randomUUID(); + try { + await this.addToAndExecutePipeline(pipelineExecutionId, sdsStrings); + } catch (e) { + printOutputMessage('Error during pipeline execution: ' + e); + throw e; + } + + // Get missing value ratio for each column + for (const [, placeholderName] of columnNameToPlaceholderMVNameMap) { + const missingValueRatio = await this.getPlaceholderValue(placeholderName, pipelineExecutionId); + if (missingValueRatio) { + missingValueRatioMap.set(placeholderName, missingValueRatio as number); + } + } + + // Get IDness for each column + for (const [, placeholderName] of columnNameToPlaceholderIDnessNameMap) { + const idness = await this.getPlaceholderValue(placeholderName, pipelineExecutionId); + if (idness) { + idnessMap.set(placeholderName, idness as number); + } + } + + // Get histogram for each column + for (const [, placeholderName] of columnNameToPlaceholderHistogramNameMap) { + const histogram = await this.getPlaceholderValue(placeholderName, pipelineExecutionId); + if (histogram) { + histogramMap.set(placeholderName, histogram as Base64Image); + } + } + + // Create profiling data + const profiling: { columnName: string; profiling: Profiling }[] = []; + for (const column of columns) { + // Base info for the top of the profiling + const missingValuesRatio = + missingValueRatioMap.get(columnNameToPlaceholderMVNameMap.get(column[1].name)!)! * 100; + + const validRatio: ProfilingDetailStatistical = { + type: 'numerical', + name: 'Valid', + value: missingValuesRatio ? (100 - missingValuesRatio).toFixed(2) + '%' : '100%', + interpretation: 'good', + }; + + const missingRatio: ProfilingDetailStatistical = { + type: 'numerical', + name: 'Missing', + value: missingValuesRatio ? missingValuesRatio.toFixed(2) + '%' : '0%', + interpretation: missingValuesRatio > 0 ? 'error' : 'default', + }; + + // If not numerical, add proper profilings according to idness results + if (column[1].type !== 'numerical') { + const idness = idnessMap.get(columnNameToPlaceholderIDnessNameMap.get(column[1].name)!)!; + const uniqueValues = idness * column[1].values.length; + + if (uniqueValues <= 3) { + // Can display each separate percentages of unique values + // Find all unique values and count them + const uniqueValueCounts = new Map(); + for (let i = 0; i < column[1].values.length; i++) { + if (column[1].values[i]) + uniqueValueCounts.set( + column[1].values[i], + (uniqueValueCounts.get(column[1].values[i]) || 0) + 1, + ); + } + + let uniqueProfilings: ProfilingDetailStatistical[] = []; + for (const [key, value] of uniqueValueCounts) { + uniqueProfilings.push({ + type: 'numerical', + name: key, + value: ((value / column[1].values.length) * 100).toFixed(2) + '%', + interpretation: 'category', + }); + } + + profiling.push({ + columnName: column[1].name, + profiling: { + validRatio, + missingRatio, + other: [ + { type: 'text', value: 'Categorical', interpretation: 'important' }, + ...uniqueProfilings, + ], + }, + }); + } else if (uniqueValues <= 10) { + // Display histogram for 4-10 unique values, has to match the condition above where histogram is generated + const histogram = histogramMap.get(columnNameToPlaceholderHistogramNameMap.get(column[1].name)!)!; + + profiling.push({ + columnName: column[1].name, + profiling: { + validRatio, + missingRatio, + other: [ + { type: 'text', value: 'Categorical', interpretation: 'important' }, + { type: 'image', value: histogram }, + ], + }, + }); + } else { + // Display only the number of unique values vs total valid values + profiling.push({ + columnName: column[1].name, + profiling: { + validRatio, + missingRatio, + other: [ + { type: 'text', value: 'Categorical', interpretation: 'important' }, + { + type: 'text', + value: uniqueValues + ' Distincts', + interpretation: 'default', + }, + { + type: 'text', + value: + Math.round( + column[1].values.length * + (1 - + (missingValueRatioMap.get( + columnNameToPlaceholderMVNameMap.get(column[1].name)!, + ) || 0)), + ) + ' Total Valids', + interpretation: 'default', + }, + ], + }, + }); + } + } else { + // Display histogram for numerical columns + const histogram = histogramMap.get(columnNameToPlaceholderHistogramNameMap.get(column[1].name)!)!; + + profiling.push({ + columnName: column[1].name, + profiling: { + validRatio, + missingRatio, + other: [ + { type: 'text', value: 'Numerical', interpretation: 'important' }, + { type: 'image', value: histogram }, + ], + }, + }); + } + } + + return profiling; + } +} diff --git a/packages/safe-ds-vscode/src/extension/eda/edaPanel.ts b/packages/safe-ds-vscode/src/extension/eda/edaPanel.ts index 2c5d2c830..5b81156ce 100644 --- a/packages/safe-ds-vscode/src/extension/eda/edaPanel.ts +++ b/packages/safe-ds-vscode/src/extension/eda/edaPanel.ts @@ -1,11 +1,10 @@ import * as vscode from 'vscode'; import { ToExtensionMessage } from '@safe-ds/eda/types/messaging.js'; import * as webviewApi from './apis/webviewApi.ts'; -import { Column, State, Table } from '@safe-ds/eda/types/state.js'; +import { State } from '@safe-ds/eda/types/state.js'; import { logOutput, printOutputMessage } from '../output.ts'; -import { messages, SafeDsServices } from '@safe-ds/lang'; - -export const undefinedPanelIdentifier = 'undefinedPanelIdentifier'; +import { SafeDsServices, ast } from '@safe-ds/lang'; +import { RunnerApi } from './apis/runnerApi.ts'; export class EDAPanel { // Map to track multiple panels @@ -18,24 +17,33 @@ export class EDAPanel { private readonly panel: vscode.WebviewPanel; private readonly extensionUri: vscode.Uri; private disposables: vscode.Disposable[] = []; - private tableIdentifier: string | undefined; - private startPipelineId: string = ''; + private tableIdentifier: string; + private tableName: string; private column: vscode.ViewColumn | undefined; private webviewListener: vscode.Disposable | undefined; private viewStateChangeListener: vscode.Disposable | undefined; + private updateHtmlDone: boolean = false; + private startPipelineExecutionId: string; + private runnerApi: RunnerApi; private constructor( panel: vscode.WebviewPanel, extensionUri: vscode.Uri, - startPipeLineId: string, - tableIdentifier?: string, + startPipelineExecutionId: string, + pipelinePath: vscode.Uri, + pipelineName: string, + pipelineNode: ast.SdsPipeline, + tableName: string, ) { + this.tableIdentifier = pipelineName + '.' + tableName; this.panel = panel; this.extensionUri = extensionUri; - this.tableIdentifier = tableIdentifier; - this.startPipelineId = startPipeLineId; + this.startPipelineExecutionId = startPipelineExecutionId; + this.runnerApi = new RunnerApi(EDAPanel.services, pipelinePath, pipelineName, pipelineNode); + this.tableName = tableName; // Set the webview's initial html content + this.updateHtmlDone = false; this._update(); // Listen for when the panel is disposed @@ -95,34 +103,35 @@ export class EDAPanel { this.disposables.push(this.webviewListener); } - public static createOrShow( + public static async createOrShow( extensionUri: vscode.Uri, context: vscode.ExtensionContext, - startPipelineId: string, - servicess: SafeDsServices, - tableIdentifier?: string, - ) { + startPipelineExecutionId: string, + services: SafeDsServices, + pipelinePath: vscode.Uri, + pipelineName: string, + pipelineNode: ast.SdsPipeline, + tableName: string, + ): Promise { EDAPanel.context = context; - EDAPanel.services = servicess; + EDAPanel.services = services; + + let tableIdentifier = pipelineName + '.' + tableName; // Set column to the active editor if it exists const column = vscode.window.activeTextEditor ? vscode.window.activeTextEditor.viewColumn : undefined; // If we already have a panel, show it. - let panel = EDAPanel.panelsMap.get(tableIdentifier ?? undefinedPanelIdentifier); + let panel = EDAPanel.panelsMap.get(tableIdentifier); if (panel) { panel.panel.reveal(panel.column); panel.tableIdentifier = tableIdentifier; - panel.startPipelineId = startPipelineId; - - // Have to update and construct state as table placeholder could've changed in code - panel._update(); - panel.constructCurrentState().then((state) => { - webviewApi.postMessage(panel!.panel.webview, { - command: 'setWebviewState', - value: state, - }); - }); + panel.startPipelineExecutionId = startPipelineExecutionId; + panel.runnerApi = new RunnerApi(services, pipelinePath, pipelineName, pipelineNode); + panel.tableName = tableName; + EDAPanel.panelsMap.set(tableIdentifier, panel); + + // TODO: Display disclaimer that data can be outdated and show refresh button return; } else { // Otherwise, create a new panel. @@ -142,19 +151,44 @@ export class EDAPanel { }, ); - const edaPanel = new EDAPanel(newPanel, extensionUri, startPipelineId, tableIdentifier); - EDAPanel.panelsMap.set(tableIdentifier ?? undefinedPanelIdentifier, edaPanel); + const edaPanel = new EDAPanel( + newPanel, + extensionUri, + startPipelineExecutionId, + pipelinePath, + pipelineName, + pipelineNode, + tableName, + ); + EDAPanel.panelsMap.set(tableIdentifier, edaPanel); edaPanel.column = column; edaPanel.panel.iconPath = { light: vscode.Uri.joinPath(edaPanel.extensionUri, 'img', 'binoculars-solid.png'), dark: vscode.Uri.joinPath(edaPanel.extensionUri, 'img', 'binoculars-solid.png'), }; - edaPanel.constructCurrentState().then((state) => { + await edaPanel.waitForUpdateHtmlDone(10000); + const stateInfo = await edaPanel.constructCurrentState(); + webviewApi.postMessage(edaPanel!.panel.webview, { + command: 'setWebviewState', + value: stateInfo.state, + }); + + // TODO: if from existing state, show disclaimer that updated data is loading and execute pipeline + history + profiling and send + + if ( + !stateInfo.fromExisting || + !stateInfo.state.table || + !stateInfo.state.table!.columns.find((c) => c[1].profiling) + ) { + const profiling = await EDAPanel.panelsMap + .get(tableIdentifier)! + .runnerApi.getProfiling(stateInfo.state.table!); + webviewApi.postMessage(edaPanel!.panel.webview, { - command: 'setWebviewState', - value: state, + command: 'setProfiling', + value: profiling, }); - }); + } } } @@ -162,22 +196,13 @@ export class EDAPanel { printOutputMessage('kill ' + tableIdentifier); let panel = EDAPanel.panelsMap.get(tableIdentifier); if (panel) { - panel.dispose(); + panel.panel.dispose(); EDAPanel.panelsMap.delete(tableIdentifier); } } - public static revive(panel: vscode.WebviewPanel, extensionUri: vscode.Uri, tableIdentifier: string) { - const existingPanel = EDAPanel.panelsMap.get(tableIdentifier); - if (existingPanel) { - existingPanel.dispose(); - } - const revivedPanel = new EDAPanel(panel, extensionUri, existingPanel?.startPipelineId ?? '', tableIdentifier); - EDAPanel.panelsMap.set(tableIdentifier, revivedPanel); - } - public dispose() { - EDAPanel.panelsMap.delete(this.tableIdentifier ?? undefinedPanelIdentifier); + EDAPanel.panelsMap.delete(this.tableIdentifier); // Clean up our panel this.panel.dispose(); @@ -194,81 +219,57 @@ export class EDAPanel { private async _update() { const webview = this.panel.webview; this.panel.webview.html = await this._getHtmlForWebview(webview); + this.updateHtmlDone = true; } + private waitForUpdateHtmlDone = (timeoutMs: number): Promise => { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + // Function to check updateHtmlDone status + const check = () => { + if (this.updateHtmlDone) { + resolve(); + } else if (Date.now() - startTime > timeoutMs) { + reject(new Error('Timeout waiting for updateHtmlDone')); + } else { + setTimeout(check, 100); // Check every 100ms + } + }; + check(); + }); + }; + private findCurrentState(): State | undefined { const existingStates = (EDAPanel.context.globalState.get('webviewState') ?? []) as State[]; return existingStates.find((s) => s.tableIdentifier === this.tableIdentifier); } - private constructCurrentState(): Promise { - return new Promise((resolve, reject) => { - const existingCurrentState = this.findCurrentState(); - if (existingCurrentState) { - printOutputMessage('Found current State.'); - resolve(existingCurrentState); - return; - } - - if (!this.tableIdentifier) { - resolve({ tableIdentifier: undefined, history: [], defaultState: true }); - return; - } + private async constructCurrentState(): Promise<{ state: State; fromExisting: boolean }> { + const existingCurrentState = this.findCurrentState(); + if (existingCurrentState) { + printOutputMessage('Found current State.'); + return { state: existingCurrentState, fromExisting: true }; + } - const placeholderValueCallback = (message: messages.PlaceholderValueMessage) => { - if (message.id !== this.startPipelineId || message.data.name !== this.tableIdentifier) { - return; - } - EDAPanel.services.runtime.Runner.removeMessageCallback(placeholderValueCallback, 'placeholder_value'); - - const pythonTableColumns = message.data.value; - const table: Table = { - totalRows: 0, - name: this.tableIdentifier, - columns: [] as Table['columns'], - appliedFilters: [] as Table['appliedFilters'], + const panel = EDAPanel.panelsMap.get(this.tableIdentifier); + if (!panel) { + throw new Error('RunnerApi panel not found.'); + } else { + const table = await panel.runnerApi.getTableByPlaceholder(this.tableName, this.startPipelineExecutionId); + if (!table) { + throw new Error('Timeout waiting for placeholder value'); + } else { + return { + state: { + tableIdentifier: panel.tableIdentifier, + history: [], + defaultState: false, + table, + }, + fromExisting: false, }; - - let i = 0; - let currentMax = 0; - for (const [columnName, columnValues] of Object.entries(pythonTableColumns)) { - if (!Array.isArray(columnValues)) { - continue; - } - if (currentMax < columnValues.length) { - currentMax = columnValues.length; - } - - const isNumerical = typeof columnValues[0] === 'number'; - const columnType = isNumerical ? 'numerical' : 'categorical'; - - const column: Column = { - name: columnName, - values: columnValues, - type: columnType, - hidden: false, - highlighted: false, - appliedFilters: [], - appliedSort: null, - profiling: { top: [], bottom: [] }, - coloredHighLow: false, - }; - table.columns.push([i++, column]); - } - table.totalRows = currentMax; - table.visibleRows = currentMax; - printOutputMessage('Got placeholder from Runner!'); - resolve({ tableIdentifier: this.tableIdentifier, history: [], defaultState: false, table }); - }; - - EDAPanel.services.runtime.Runner.addMessageCallback(placeholderValueCallback, 'placeholder_value'); - printOutputMessage('Getting placeholder from Runner ...'); - EDAPanel.services.runtime.Runner.sendMessageToPythonServer( - messages.createPlaceholderQueryMessage(this.startPipelineId, this.tableIdentifier), - ); - - setTimeout(() => reject(new Error('Timeout waiting for placeholder value')), 30000); - }); + } + } } private async _getHtmlForWebview(webview: vscode.Webview) { @@ -310,7 +311,6 @@ export class EDAPanel { diff --git a/packages/safe-ds-vscode/src/extension/mainClient.ts b/packages/safe-ds-vscode/src/extension/mainClient.ts index 0c0bf3987..c3ceb5d4b 100644 --- a/packages/safe-ds-vscode/src/extension/mainClient.ts +++ b/packages/safe-ds-vscode/src/extension/mainClient.ts @@ -6,15 +6,19 @@ import { ast, createSafeDsServices, getModuleMembers, messages, SafeDsServices } import { NodeFileSystem } from 'langium/node'; import { getSafeDSOutputChannel, initializeLog, logError, logOutput, printOutputMessage } from './output.js'; import crypto from 'crypto'; -import { LangiumDocument, URI } from 'langium'; -import { EDAPanel, undefinedPanelIdentifier } from './eda/edaPanel.ts'; +import { LangiumDocument, URI, AstUtils, AstNode } from 'langium'; +import { EDAPanel } from './eda/edaPanel.ts'; import { dumpDiagnostics } from './commands/dumpDiagnostics.js'; import { openDiagnosticsDumps } from './commands/openDiagnosticsDumps.js'; +import { Range } from 'vscode-languageclient'; let client: LanguageClient; let services: SafeDsServices; -let lastFinishedPipelineId: string | undefined; -let lastSuccessfulPlaceholderName: string | undefined; +let lastFinishedPipelineExecutionId: string | undefined; +let lastSuccessfulPipelineName: string | undefined; +let lastSuccessfulTableName: string | undefined; +let lastSuccessfulPipelinePath: vscode.Uri | undefined; +let lastSuccessfulPipelineNode: ast.SdsPipeline | undefined; // This function is called when the extension is activated. export const activate = async function (context: vscode.ExtensionContext) { @@ -86,7 +90,7 @@ const startLanguageClient = function (context: vscode.ExtensionContext): Languag return result; }; -const acceptRunRequests = function (context: vscode.ExtensionContext) { +const acceptRunRequests = async function (context: vscode.ExtensionContext) { // Register logging message callbacks registerMessageLoggingCallbacks(); // Register VS Code Entry Points @@ -145,7 +149,7 @@ const registerMessageLoggingCallbacks = function () { }; const registerVSCodeCommands = function (context: vscode.ExtensionContext) { - const registerCommandWithCheck = (commandId: string, callback: (...args: any[]) => any) => { + const registerCommandWithCheck = (commandId: string, callback: (...args: any[]) => Promise) => { return vscode.commands.registerCommand(commandId, (...args: any[]) => { if (!services.runtime.Runner.isPythonServerAvailable()) { vscode.window.showErrorMessage('Extension not fully started yet.'); @@ -163,7 +167,7 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { context.subscriptions.push(vscode.commands.registerCommand('safe-ds.runPipelineFile', commandRunPipelineFile)); context.subscriptions.push( - registerCommandWithCheck('safe-ds.runEdaFromContext', () => { + registerCommandWithCheck('safe-ds.runEdaFromContext', async () => { const editor = vscode.window.activeTextEditor; if (editor) { const position = editor.selection.active; @@ -178,8 +182,34 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { vscode.window.showErrorMessage('No .sdspipe file selected!'); return; } + + // Getting of pipeline name + const document = await getPipelineDocument(editor.document.uri); + if (!document) { + vscode.window.showErrorMessage('Internal error'); + return; + } + + // Find node of placeholder + let placeholderNode = findPlaceholderNode(document, range); + + if (!placeholderNode) { + vscode.window.showErrorMessage('Internal error'); + return; + } + + // Get pipeline container + const container = AstUtils.getContainerOfType(placeholderNode, ast.isSdsPipeline); + if (!container) { + vscode.window.showErrorMessage('Internal error'); + return; + } + + const pipelineName = container.name; + const pipelineNode = container; + // gen custom id for pipeline - const pipelineId = crypto.randomUUID(); + const pipelineExecutionId = crypto.randomUUID(); let loadingInProgress = true; // Flag to track loading status // Show progress indicator @@ -210,25 +240,34 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { `Placeholder was calculated (${message.id}): ${message.data.name} of type ${message.data.type}`, ); if ( - message.id === pipelineId && + message.id === pipelineExecutionId && message.data.type === 'Table' && message.data.name === requestedPlaceholderName ) { - lastFinishedPipelineId = pipelineId; - lastSuccessfulPlaceholderName = requestedPlaceholderName; + lastFinishedPipelineExecutionId = pipelineExecutionId; + lastSuccessfulPipelinePath = editor.document.uri; + lastSuccessfulTableName = requestedPlaceholderName; + lastSuccessfulPipelineName = pipelineName; + lastSuccessfulPipelineNode = pipelineNode; EDAPanel.createOrShow( context.extensionUri, context, - pipelineId, + pipelineExecutionId, services, + editor.document.uri, + pipelineName, + pipelineNode, message.data.name, ); services.runtime.Runner.removeMessageCallback(placeholderTypeCallback, 'placeholder_type'); cleanupLoadingIndication(); - } else if (message.id === pipelineId && message.data.name !== requestedPlaceholderName) { + } else if ( + message.id === pipelineExecutionId && + message.data.name !== requestedPlaceholderName + ) { return; - } else if (message.id === pipelineId) { - lastFinishedPipelineId = pipelineId; + } else if (message.id === pipelineExecutionId) { + lastFinishedPipelineExecutionId = pipelineExecutionId; vscode.window.showErrorMessage(`Selected placeholder is not of type 'Table'.`); services.runtime.Runner.removeMessageCallback(placeholderTypeCallback, 'placeholder_type'); cleanupLoadingIndication(); @@ -239,11 +278,11 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { const runtimeProgressCallback = function (message: messages.RuntimeProgressMessage) { printOutputMessage(`Runner-Progress (${message.id}): ${message.data}`); if ( - message.id === pipelineId && + message.id === pipelineExecutionId && message.data === 'done' && - lastFinishedPipelineId !== pipelineId + lastFinishedPipelineExecutionId !== pipelineExecutionId ) { - lastFinishedPipelineId = pipelineId; + lastFinishedPipelineExecutionId = pipelineExecutionId; vscode.window.showErrorMessage(`Selected text is not a placeholder!`); services.runtime.Runner.removeMessageCallback(runtimeProgressCallback, 'runtime_progress'); cleanupLoadingIndication(); @@ -252,8 +291,11 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { services.runtime.Runner.addMessageCallback(runtimeProgressCallback, 'runtime_progress'); const runtimeErrorCallback = function (message: messages.RuntimeErrorMessage) { - if (message.id === pipelineId && lastFinishedPipelineId !== pipelineId) { - lastFinishedPipelineId = pipelineId; + if ( + message.id === pipelineExecutionId && + lastFinishedPipelineExecutionId !== pipelineExecutionId + ) { + lastFinishedPipelineExecutionId = pipelineExecutionId; vscode.window.showErrorMessage(`Pipeline ran into an Error!`); services.runtime.Runner.removeMessageCallback(runtimeErrorCallback, 'runtime_error'); cleanupLoadingIndication(); @@ -261,9 +303,9 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { }; services.runtime.Runner.addMessageCallback(runtimeErrorCallback, 'runtime_error'); - runPipelineFile(editor.document.uri, pipelineId); + runPipelineFile(editor.document.uri, pipelineExecutionId, pipelineName, requestedPlaceholderName); } else { - EDAPanel.createOrShow(context.extensionUri, context, '', services, undefined); + vscode.window.showErrorMessage('No placeholder selected!'); } } else { vscode.window.showErrorMessage('No ative text editor!'); @@ -274,14 +316,27 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { context.subscriptions.push( vscode.commands.registerCommand('safe-ds.refreshWebview', () => { - EDAPanel.kill(lastSuccessfulPlaceholderName ? lastSuccessfulPlaceholderName : undefinedPanelIdentifier); + if ( + !lastSuccessfulPipelinePath || + !lastFinishedPipelineExecutionId || + !lastSuccessfulPipelineName || + !lastSuccessfulTableName || + !lastSuccessfulPipelineNode + ) { + vscode.window.showErrorMessage('No EDA Panel to refresh!'); + return; + } + EDAPanel.kill(lastSuccessfulPipelineName! + '.' + lastSuccessfulTableName!); setTimeout(() => { EDAPanel.createOrShow( context.extensionUri, context, - '', + lastFinishedPipelineExecutionId!, services, - lastSuccessfulPlaceholderName ? lastSuccessfulPlaceholderName : undefinedPanelIdentifier, + lastSuccessfulPipelinePath!, + lastSuccessfulPipelineName!, + lastSuccessfulPipelineNode!, + lastSuccessfulTableName!, ); }, 100); setTimeout(() => { @@ -291,7 +346,38 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { ); }; -const runPipelineFile = async function (filePath: vscode.Uri | undefined, pipelineId: string) { +const runPipelineFile = async function ( + filePath: vscode.Uri | undefined, + pipelineExecutionId: string, + knownPipelineName?: string, + placeholderName?: string, +) { + const document = await getPipelineDocument(filePath); + + if (document) { + // Run it + let pipelineName; + if (!knownPipelineName) { + const firstPipeline = getModuleMembers(document.parseResult.value).find(ast.isSdsPipeline); + if (firstPipeline === undefined) { + logError('Cannot execute: no pipeline found'); + vscode.window.showErrorMessage('The current file cannot be executed, as no pipeline could be found.'); + return; + } + pipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name; + } else { + pipelineName = knownPipelineName; + } + + printOutputMessage(`Launching Pipeline (${pipelineExecutionId}): ${filePath} - ${pipelineName}`); + + await services.runtime.Runner.executePipeline(pipelineExecutionId, document, pipelineName, placeholderName); + } +}; + +export const getPipelineDocument = async function ( + filePath: vscode.Uri | undefined, +): Promise { let pipelinePath = filePath; // Allow execution via command menu if (!pipelinePath && vscode.window.activeTextEditor) { @@ -334,7 +420,7 @@ const runPipelineFile = async function (filePath: vscode.Uri | undefined, pipeli vscode.window.showErrorMessage(validationErrorMessage); return; } - // Run it + let mainDocument; if (!services.shared.workspace.LangiumDocuments.hasDocument(pipelinePath)) { mainDocument = await services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath); @@ -347,17 +433,7 @@ const runPipelineFile = async function (filePath: vscode.Uri | undefined, pipeli mainDocument = await services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath); } - const firstPipeline = getModuleMembers(mainDocument.parseResult.value).find(ast.isSdsPipeline); - if (firstPipeline === undefined) { - logError('Cannot execute: no pipeline found'); - vscode.window.showErrorMessage('The current file cannot be executed, as no pipeline could be found.'); - return; - } - const mainPipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name; - - printOutputMessage(`Launching Pipeline (${pipelineId}): ${pipelinePath} - ${mainPipelineName}`); - - await services.runtime.Runner.executePipeline(pipelineId, mainDocument, mainPipelineName); + return mainDocument; }; const commandRunPipelineFile = async function (filePath: vscode.Uri | undefined) { @@ -413,3 +489,31 @@ const registerVSCodeWatchers = function () { } }); }; + +const isRangeEqual = function (lhs: Range, rhs: Range): boolean { + return ( + lhs.start.character === rhs.start.character && + lhs.start.line === rhs.start.line && + lhs.end.character === rhs.end.character && + lhs.end.line === rhs.end.line + ); +}; + +const findPlaceholderNode = function (document: LangiumDocument, range: vscode.Range): AstNode | undefined { + let placeholderNode: AstNode | undefined; + const module = document.parseResult.value as ast.SdsModule; + for (const node of AstUtils.streamAllContents(module, { range })) { + // Entire node matches the range + const actualRange = node.$cstNode?.range; + if (actualRange && isRangeEqual(actualRange, range)) { + placeholderNode = node; + } + + // The node has a name node that matches the range + const actualNameRange = services.references.NameProvider.getNameNode(node)?.range; + if (actualNameRange && isRangeEqual(actualNameRange, range)) { + placeholderNode = node; + } + } + return placeholderNode; +};