From 50aac72a6c9d6d073be5555e54c7d903f680e63a Mon Sep 17 00:00:00 2001 From: Ramon Vermeulen Date: Tue, 6 Aug 2024 21:22:32 +0200 Subject: [PATCH 1/5] WIP on preview data functionality --- .../dbtToolkit/services/DbtCommandExecutorService.kt | 8 ++++---- .../dbtToolkit/services/ProcessOutputHandlerService.kt | 3 ++- .../dbtToolkit/ui/DbtToolkitMainToolWindow.kt | 2 ++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt index ba408252..df3b1eff 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt @@ -29,21 +29,21 @@ class DbtCommandExecutorService(private var project: Project) { } @Synchronized - fun executeCommand(command: List) { + fun executeCommand(command: List): String { val latch = CountDownLatch(1) - + var output = "" showLoadingIndicator("Executing dbt ${command.joinToString(" ")}...") { try { if (!venvInitializerService.isInitialized) { venvInitializerService.initializeEnvironment() } val process = processExecutorService.executeCommand(command) - outputHandler.handleOutput(process) + output = outputHandler.handleOutput(process) } finally { latch.countDown() } } - latch.await() + return output } } diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt index ad6c0c12..eba1f9e6 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt @@ -14,7 +14,7 @@ class ProcessOutputHandlerService(project: Project) { private val loggingService = project.service() private val notificationService = project.service() - fun handleOutput(process: Process) { + fun handleOutput(process: Process): String { val exitCode = if (process.waitFor(settings.state.settingsDbtCommandTimeout, TimeUnit.SECONDS)) { process.exitValue() @@ -41,5 +41,6 @@ class ProcessOutputHandlerService(project: Project) { } else { loggingService.log(stdout, ConsoleViewContentType.NORMAL_OUTPUT) } + return output } } diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt index 42a5d9a3..2bc7795b 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt @@ -5,6 +5,7 @@ import com.github.ramonvermeulen.dbtToolkit.ui.console.ConsoleOutputPanel import com.github.ramonvermeulen.dbtToolkit.ui.panels.CompiledSqlPanel import com.github.ramonvermeulen.dbtToolkit.ui.panels.DocsPanel import com.github.ramonvermeulen.dbtToolkit.ui.panels.LineagePanel +import com.github.ramonvermeulen.dbtToolkit.ui.panels.PreviewDataPanel import com.intellij.openapi.components.service import com.intellij.openapi.project.DumbAware import com.intellij.openapi.project.Project @@ -35,6 +36,7 @@ class DbtToolkitMainToolWindow : ToolWindowFactory, DumbAware { "lineage" to PanelInfo({ LineagePanel(project) }, false), "docs" to PanelInfo({ DocsPanel(project) }, true), "compiled sql" to PanelInfo({ CompiledSqlPanel(project) }, true), + "preview model" to PanelInfo({ PreviewDataPanel(project) }, true), "console (read-only)" to PanelInfo({ ConsoleOutputPanel(project) }, false), ) From 410ddddc886ead92eb9110c168fbcbf6da26b20d Mon Sep 17 00:00:00 2001 From: Ramon Vermeulen Date: Wed, 7 Aug 2024 15:24:16 +0200 Subject: [PATCH 2/5] WIP on preview data functionality --- .../dbtToolkit/ui/panels/PreviewDataPanel.kt | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt new file mode 100644 index 00000000..3910cc6c --- /dev/null +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt @@ -0,0 +1,97 @@ +package com.github.ramonvermeulen.dbtToolkit.ui.panels + +import com.github.ramonvermeulen.dbtToolkit.services.ActiveFileListener +import com.github.ramonvermeulen.dbtToolkit.services.ActiveFileService +import com.github.ramonvermeulen.dbtToolkit.services.DbtCommandExecutorService +import com.github.ramonvermeulen.dbtToolkit.services.DbtToolkitSettingsService +import com.github.ramonvermeulen.dbtToolkit.ui.IdeaPanel +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.components.service +import com.intellij.openapi.editor.EditorFactory +import com.intellij.openapi.fileEditor.FileDocumentManager +import com.intellij.openapi.fileEditor.FileEditorManager +import com.intellij.openapi.fileTypes.FileTypeManager +import com.intellij.openapi.project.Project +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.openapi.vfs.VirtualFileManager +import java.awt.BorderLayout +import java.nio.file.Paths +import javax.swing.JButton +import javax.swing.JComponent +import javax.swing.JPanel +import javax.swing.SwingUtilities + +class PreviewDataPanel(private val project: Project): IdeaPanel, Disposable, ActiveFileListener { + private val settings = project.service() + private val dbtCommandExecutorService = project.service() + private val mainPanel = JPanel(BorderLayout()) + private val previewDataButton = JButton("Preview Data") + private var document = EditorFactory.getInstance().createDocument("") + private var activeFile: VirtualFile? = null + + init { + project.messageBus.connect().subscribe(ActiveFileService.TOPIC, this) + val fileType = FileTypeManager.getInstance().getFileTypeByExtension("txt") + val editor = EditorFactory.getInstance().createEditor(document, project, fileType, true) + val editorTextField = editor.component + // to set the initial file, since the subscription is only set-up after + // opening the panel (lazy) for the first time + activeFileChanged(FileEditorManager.getInstance(project).selectedFiles.firstOrNull()) + previewDataButton.addActionListener { handleRecompileButtonClick() } + mainPanel.add(previewDataButton, BorderLayout.NORTH) + mainPanel.add(editorTextField, BorderLayout.CENTER) + } + + private fun handleRecompileButtonClick() { + SwingUtilities.invokeLater { + previewDataButton.isEnabled = false + previewDataButton.text = "Sending query to DWH..." + } + ApplicationManager.getApplication().executeOnPooledThread { + try { + if (activeFile != null) { + // save the active file on disk before sending Query, IDE can have an open buffer + ApplicationManager.getApplication().invokeLater { + ApplicationManager.getApplication().runWriteAction { + FileDocumentManager.getInstance().getDocument(activeFile!!).let { + FileDocumentManager.getInstance().saveDocument(it!!) + } + } + } + } + } finally { + previewData() + } + } + } + + override fun activeFileChanged(file: VirtualFile?) { + if (file != null && file.extension == "sql") { + activeFile = file + } + } + + private fun previewData() { + ApplicationManager.getApplication().executeOnPooledThread { + val output = dbtCommandExecutorService.executeCommand( + listOf("show", "--no-populate-cache", "--select", activeFile!!.nameWithoutExtension, "--limit", "10") + ) + val data = output.split("\n").takeLast(16).joinToString("\n") + SwingUtilities.invokeLater { + ApplicationManager.getApplication().runWriteAction { + document.setText(data) + previewDataButton.isEnabled = true + previewDataButton.text = "Preview Data" + } + } + } + } + + override fun getContent(): JComponent { + return mainPanel + } + + override fun dispose() { + // Implement your dispose logic here + }} \ No newline at end of file From 60b1359b89d1e99f54d8dc58884317b5cc87219e Mon Sep 17 00:00:00 2001 From: Ramon Vermeulen Date: Wed, 7 Aug 2024 15:49:32 +0200 Subject: [PATCH 3/5] PreviewDataPanel implemented + some QoL improvements --- .../ramonvermeulen/dbtToolkit/Constants.kt | 1 + .../listeners/DbtToolkitFileListener.kt | 8 +-- .../services/DbtCommandExecutorService.kt | 4 +- .../services/ProcessOutputHandlerService.kt | 4 +- .../dbtToolkit/ui/panels/CompiledSqlPanel.kt | 12 +++++ .../dbtToolkit/ui/panels/LineagePanel.kt | 3 +- .../dbtToolkit/ui/panels/PreviewDataPanel.kt | 53 +++++++++++++------ 7 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/Constants.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/Constants.kt index 4573abed..e3ae2f53 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/Constants.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/Constants.kt @@ -3,6 +3,7 @@ package com.github.ramonvermeulen.dbtToolkit import com.intellij.openapi.util.Key val JS_TRIGGERED_KEY: Key = Key.create("JS_TRIGGERED_KEY") +val SUPPORTED_LINEAGE_EXTENSIONS = setOf("sql", "csv") const val DBT_DOCS_FILE = "index.html" const val DBT_MANIFEST_FILE = "manifest.json" const val DBT_CATALOG_FILE = "catalog.json" diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt index 1b5f6e60..c01a789d 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt @@ -10,13 +10,7 @@ import com.intellij.openapi.vfs.VirtualFile class DbtToolkitFileListener(project: Project) : FileEditorManagerListener { private val activeFileService = project.service() - companion object { - val SUPPORTED_EXTENSIONS = setOf("sql", "csv") - } - override fun selectionChanged(event: FileEditorManagerEvent) { - if (SUPPORTED_EXTENSIONS.contains(event.newFile?.extension)) { - activeFileService.setActiveFile(event.newFile as VirtualFile) - } + activeFileService.setActiveFile(event.newFile as VirtualFile) } } diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt index df3b1eff..9de78233 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/DbtCommandExecutorService.kt @@ -29,9 +29,9 @@ class DbtCommandExecutorService(private var project: Project) { } @Synchronized - fun executeCommand(command: List): String { + fun executeCommand(command: List): Pair { val latch = CountDownLatch(1) - var output = "" + var output: Pair = Pair(-1, "") showLoadingIndicator("Executing dbt ${command.joinToString(" ")}...") { try { if (!venvInitializerService.isInitialized) { diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt index eba1f9e6..c9ed6de4 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ProcessOutputHandlerService.kt @@ -14,7 +14,7 @@ class ProcessOutputHandlerService(project: Project) { private val loggingService = project.service() private val notificationService = project.service() - fun handleOutput(process: Process): String { + fun handleOutput(process: Process): Pair { val exitCode = if (process.waitFor(settings.state.settingsDbtCommandTimeout, TimeUnit.SECONDS)) { process.exitValue() @@ -41,6 +41,6 @@ class ProcessOutputHandlerService(project: Project) { } else { loggingService.log(stdout, ConsoleViewContentType.NORMAL_OUTPUT) } - return output + return Pair(exitCode, output) } } diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/CompiledSqlPanel.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/CompiledSqlPanel.kt index e750f8eb..b7bcbe02 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/CompiledSqlPanel.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/CompiledSqlPanel.kt @@ -78,6 +78,18 @@ class CompiledSqlPanel(project: Project) : IdeaPanel, Disposable, ActiveFileList activeFile = file val compiledFile = findCompiledFile(file) displayCompiledFile(compiledFile) + SwingUtilities.invokeLater { + recompileButton.isEnabled = true + recompileButton.text = "Re-compile model" + } + } else { + SwingUtilities.invokeLater { + recompileButton.isEnabled = false + recompileButton.text = "Open a SQL file" + ApplicationManager.getApplication().runWriteAction { + document.setText("") + } + } } } diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/LineagePanel.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/LineagePanel.kt index 0ce97836..2cbda715 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/LineagePanel.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/LineagePanel.kt @@ -5,6 +5,7 @@ import com.github.ramonvermeulen.dbtToolkit.LINEAGE_PANEL_APP_DIR_NAME import com.github.ramonvermeulen.dbtToolkit.LINEAGE_PANEL_CSS import com.github.ramonvermeulen.dbtToolkit.LINEAGE_PANEL_INDEX import com.github.ramonvermeulen.dbtToolkit.LINEAGE_PANEL_JS +import com.github.ramonvermeulen.dbtToolkit.SUPPORTED_LINEAGE_EXTENSIONS import com.github.ramonvermeulen.dbtToolkit.models.LineageInfo import com.github.ramonvermeulen.dbtToolkit.models.toJson import com.github.ramonvermeulen.dbtToolkit.services.ActiveFileListener @@ -161,7 +162,7 @@ class LineagePanel(private val project: Project) : // subscribers override fun activeFileChanged(file: VirtualFile?) { ApplicationManager.getApplication().executeOnPooledThread { - if (file != null) { + if (file != null && SUPPORTED_LINEAGE_EXTENSIONS.contains(file.extension)) { activeFile = file refreshLineageInfo(file, false) } diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt index 3910cc6c..b82ad974 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt @@ -3,7 +3,6 @@ package com.github.ramonvermeulen.dbtToolkit.ui.panels import com.github.ramonvermeulen.dbtToolkit.services.ActiveFileListener import com.github.ramonvermeulen.dbtToolkit.services.ActiveFileService import com.github.ramonvermeulen.dbtToolkit.services.DbtCommandExecutorService -import com.github.ramonvermeulen.dbtToolkit.services.DbtToolkitSettingsService import com.github.ramonvermeulen.dbtToolkit.ui.IdeaPanel import com.intellij.openapi.Disposable import com.intellij.openapi.application.ApplicationManager @@ -14,16 +13,13 @@ import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.fileTypes.FileTypeManager import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.VirtualFile -import com.intellij.openapi.vfs.VirtualFileManager import java.awt.BorderLayout -import java.nio.file.Paths import javax.swing.JButton import javax.swing.JComponent import javax.swing.JPanel import javax.swing.SwingUtilities -class PreviewDataPanel(private val project: Project): IdeaPanel, Disposable, ActiveFileListener { - private val settings = project.service() +class PreviewDataPanel(private val project: Project) : IdeaPanel, Disposable, ActiveFileListener { private val dbtCommandExecutorService = project.service() private val mainPanel = JPanel(BorderLayout()) private val previewDataButton = JButton("Preview Data") @@ -69,20 +65,46 @@ class PreviewDataPanel(private val project: Project): IdeaPanel, Disposable, Act override fun activeFileChanged(file: VirtualFile?) { if (file != null && file.extension == "sql") { activeFile = file + SwingUtilities.invokeLater { + previewDataButton.isEnabled = true + previewDataButton.text = "Preview Data" + } + } else { + SwingUtilities.invokeLater { + previewDataButton.isEnabled = false + previewDataButton.text = "Open a SQL file" + ApplicationManager.getApplication().runWriteAction { + document.setText("") + } + } } } private fun previewData() { ApplicationManager.getApplication().executeOnPooledThread { - val output = dbtCommandExecutorService.executeCommand( - listOf("show", "--no-populate-cache", "--select", activeFile!!.nameWithoutExtension, "--limit", "10") - ) - val data = output.split("\n").takeLast(16).joinToString("\n") - SwingUtilities.invokeLater { - ApplicationManager.getApplication().runWriteAction { - document.setText(data) - previewDataButton.isEnabled = true - previewDataButton.text = "Preview Data" + val output = + dbtCommandExecutorService.executeCommand( + listOf("show", "--no-populate-cache", "--select", activeFile!!.nameWithoutExtension, "--limit", "10"), + ) + if (output.first == 0) { + val data = output.second.split("\n").takeLast(16).joinToString("\n").trimEnd() + SwingUtilities.invokeLater { + ApplicationManager.getApplication().runWriteAction { + document.setText(data) + previewDataButton.isEnabled = true + previewDataButton.text = "Preview Data" + } + } + } else { + SwingUtilities.invokeLater { + ApplicationManager.getApplication().runWriteAction { + document.setText( + "Error occured during execution of \"dbt show\" command. " + + "Please check the logs in the console tab.", + ) + previewDataButton.isEnabled = true + previewDataButton.text = "Preview Data" + } } } } @@ -94,4 +116,5 @@ class PreviewDataPanel(private val project: Project): IdeaPanel, Disposable, Act override fun dispose() { // Implement your dispose logic here - }} \ No newline at end of file + } +} From a707b95647489e55407e217e383fc40f52920649 Mon Sep 17 00:00:00 2001 From: Ramon Vermeulen Date: Wed, 7 Aug 2024 15:55:44 +0200 Subject: [PATCH 4/5] change panel name --- .../ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt | 2 +- .../ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt index 2bc7795b..29257583 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt @@ -36,7 +36,7 @@ class DbtToolkitMainToolWindow : ToolWindowFactory, DumbAware { "lineage" to PanelInfo({ LineagePanel(project) }, false), "docs" to PanelInfo({ DocsPanel(project) }, true), "compiled sql" to PanelInfo({ CompiledSqlPanel(project) }, true), - "preview model" to PanelInfo({ PreviewDataPanel(project) }, true), + "preview data" to PanelInfo({ PreviewDataPanel(project) }, true), "console (read-only)" to PanelInfo({ ConsoleOutputPanel(project) }, false), ) diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt index b82ad974..7f3c90f7 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/panels/PreviewDataPanel.kt @@ -19,7 +19,7 @@ import javax.swing.JComponent import javax.swing.JPanel import javax.swing.SwingUtilities -class PreviewDataPanel(private val project: Project) : IdeaPanel, Disposable, ActiveFileListener { +class PreviewDataPanel(project: Project) : IdeaPanel, Disposable, ActiveFileListener { private val dbtCommandExecutorService = project.service() private val mainPanel = JPanel(BorderLayout()) private val previewDataButton = JButton("Preview Data") From 3b9b6b7f4930aa340aac19c5d931150192514e58 Mon Sep 17 00:00:00 2001 From: Ramon Vermeulen Date: Wed, 7 Aug 2024 19:53:38 +0200 Subject: [PATCH 5/5] resolve tests --- .../dbtToolkit/listeners/DbtToolkitFileListener.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt index c01a789d..60c6f4ae 100644 --- a/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt +++ b/src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/listeners/DbtToolkitFileListener.kt @@ -11,6 +11,8 @@ class DbtToolkitFileListener(project: Project) : FileEditorManagerListener { private val activeFileService = project.service() override fun selectionChanged(event: FileEditorManagerEvent) { - activeFileService.setActiveFile(event.newFile as VirtualFile) + if (event.newFile != null) { + activeFileService.setActiveFile(event.newFile as VirtualFile) + } } }